diff --git a/.github/workflows/fortran-formatting.yaml b/.github/workflows/TMP_OFF/fortran-formatting.yaml similarity index 100% rename from .github/workflows/fortran-formatting.yaml rename to .github/workflows/TMP_OFF/fortran-formatting.yaml diff --git a/.github/workflows/capgen_unit_tests.yaml b/.github/workflows/capgen_unit_tests.yaml deleted file mode 100644 index 4b8598f0..00000000 --- a/.github/workflows/capgen_unit_tests.yaml +++ /dev/null @@ -1,63 +0,0 @@ -name: Capgen Unit Tests - -on: - workflow_dispatch: - pull_request: - branches: [develop] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - unit_tests: - strategy: - matrix: - os: [ubuntu-22.04] - fortran-compiler: [gfortran-9, gfortran-10, gfortran-11, gfortran-12] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v3 - - name: update repos and install dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - build-essential \ - libopenmpi-dev \ - ${{matrix.fortran-compiler}} \ - cmake \ - python3 \ - git \ - libxml2-utils - python -m pip install --upgrade pip - pip install pytest - which xmllint - xmllint --version - which pytest - - - name: Build the framework - run: | - cmake --fresh -S. -B./build -DOPENMP=ON -DCCPP_FRAMEWORK_ENABLE_TESTS=ON - cd build - make - - - name: Run unit tests - run: | - cd build - ctest --rerun-failed --output-on-failure . --verbose - - - name: Run python tests - run: | - BUILD_DIR=./build \ - PYTHONPATH=test/:scripts/ \ - pytest \ - test/capgen_test/capgen_test_reports.py \ - test/advection_test/advection_test_reports.py \ - test/ddthost_test/ddthost_test_reports.py \ - test/var_compatibility_test/var_compatibility_test_reports.py - - - name: Run Fortran to metadata test - run: cd test && ./test_fortran_to_metadata.sh - - - name: Run offline metadata parser test - run: cd test && ./test_offline_metadata_checker.sh diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml new file mode 100644 index 00000000..3d64ef76 --- /dev/null +++ b/.github/workflows/end-to-end-tests.yaml @@ -0,0 +1,62 @@ +name: capgen end-to-end tests + +on: + workflow_dispatch: + pull_request: + branches: [develop] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + end-to-end-tests: + strategy: + matrix: + os: [ubuntu-22.04] + fortran-compiler: [gfortran-9, gfortran-10, gfortran-11, gfortran-12] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.12 + uses: actions/setup-python@v4 + with: + python-version: 3.12 + - name: Update repos and install dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + build-essential \ + libopenmpi-dev \ + ${{matrix.fortran-compiler}} \ + cmake \ + python3 \ + git \ + libxml2-utils + python -m pip install --upgrade pip + pip install pytest + which xmllint + xmllint --version + which pytest + - name: Run end-to-end tests + run: | + cmake --fresh -S./end-to-end-tests -B./build + cd build + make + ctest --rerun-failed --output-on-failure . + +# - name: Run python tests +# run: | +# BUILD_DIR=./build \ +# PYTHONPATH=test/:scripts/ \ +# pytest \ +# test/capgen_test/capgen_test_reports.py \ +# test/advection_test/advection_test_reports.py \ +# test/ddthost_test/ddthost_test_reports.py \ +# test/var_compatibility_test/var_compatibility_test_reports.py +# +# - name: Run Fortran to metadata test +# run: cd test && ./test_fortran_to_metadata.sh +# +# - name: Run offline metadata parser test +# run: cd test && ./test_offline_metadata_checker.sh diff --git a/.github/workflows/prebuild.yaml b/.github/workflows/prebuild.yaml deleted file mode 100644 index 6ef3841c..00000000 --- a/.github/workflows/prebuild.yaml +++ /dev/null @@ -1,68 +0,0 @@ -name: ccpp-prebuild - -on: - pull_request: - branches: [develop] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - unit-tests: - - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8","3.9","3.10","3.11","3.12"] - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Install open mpi - run: | - wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz - tar -xvf v4.1.6.tar.gz - cd ompi-4.1.6 - ./autogen.pl - ./configure --prefix=/home/runner/ompi-4.1.6 - make -j4 - make install - echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV - echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV - - name: ccpp-prebuild unit tests - run: | - export PYTHONPATH=$(pwd)/scripts:$(pwd)/scripts/parse_tools - cd test_prebuild - pytest - # No longer possible because of https://github.com/NCAR/ccpp-framework/pull/659 - #- name: ccpp-prebuild blocked data tests - # run: | - # cd test_prebuild/test_blocked_data - # python3 ../../scripts/ccpp_prebuild.py --config=ccpp_prebuild_config.py --builddir=build - # cd build - # cmake .. - # make - # ./test_blocked_data.x - - name: ccpp-prebuild chunked data tests - run: | - cd test_prebuild/test_chunked_data - python3 ../../scripts/ccpp_prebuild.py --config=ccpp_prebuild_config.py --builddir=build - cd build - cmake .. - make - ./test_chunked_data.x - diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml deleted file mode 100644 index b8508f2a..00000000 --- a/.github/workflows/python.yaml +++ /dev/null @@ -1,56 +0,0 @@ -name: Python package - -on: - workflow_dispatch: - pull_request: - branches: [develop] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - libxml2-utils - python -m pip install --upgrade pip - pip install pytest - which pytest - - - name: Test with pytest - run: | - export PYTHONPATH=$(pwd)/scripts:$(pwd)/scripts/parse_tools - which xmllint - xmllint --version - pytest -v test/ - - - name: Test with pytest using bad xmllint (xmllint wrapper) - run: | - export XMLLINT_REAL=$(which xmllint) - export PYTHONPATH=$(pwd)/scripts:$(pwd)/scripts/parse_tools - export PATH=$(pwd)/test/unit_tests/xmllint_wrapper:${PATH} - export | grep PATH - which xmllint - xmllint --version - pytest -v test/ - - - name: Test with doctest - run: | - export PYTHONPATH=$(pwd)/scripts:$(pwd)/scripts/parse_tools - which xmllint - xmllint --version - pytest -v scripts/ --doctest-modules diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml new file mode 100644 index 00000000..7a87f6d7 --- /dev/null +++ b/.github/workflows/unit-tests.yaml @@ -0,0 +1,48 @@ +name: capgen unit tests + +on: + workflow_dispatch: + pull_request: + branches: [develop] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + + unit-tests: + name: capgen unit tests + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.9', '3.10', '3.11', '3.12'] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Update repos and install dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + build-essential \ + libopenmpi-dev \ + ${{matrix.fortran-compiler}} \ + cmake \ + python3 \ + git \ + libxml2-utils + python -m pip install --upgrade pip + pip install pytest + which xmllint + xmllint --version + which pytest + - name: Run capgen unit tests + run: | + pytest -v unit-tests/ + - name: Run capgen module doctests + run: | + PYTHONPATH=capgen pytest --doctest-modules capgen diff --git a/CMakeLists.txt b/CMakeLists.txt deleted file mode 100644 index c0aa6cee..00000000 --- a/CMakeLists.txt +++ /dev/null @@ -1,122 +0,0 @@ -cmake_minimum_required(VERSION 3.18) - -project(ccpp_framework - VERSION 5.0.0 - LANGUAGES Fortran) - -include(cmake/ccpp_capgen.cmake) - -#------------------------------------------------------------------------------ -# Set package definitions -set(PACKAGE "ccpp-framework") -string(TIMESTAMP YEAR "%Y") - -option(OPENMP "Enable OpenMP support for the framework" OFF) -option(BUILD_SHARED_LIBS "Build using shared libraries" ON) -option(CCPP_FRAMEWORK_BUILD_DOCUMENTATION - "Create and install the HTML documentation (requires Doxygen)" OFF) -option(CCPP_FRAMEWORK_ENABLE_TESTS "Enable building/running CCPP regression tests" OFF) -option(CCPP_RUN_ADVECTION_TEST "Enable advection regression test" OFF) -option(CCPP_RUN_CAPGEN_TEST "Enable capgen regression test" OFF) -option(CCPP_RUN_DDT_HOST_TEST "Enable ddt host regression test" OFF) -option(CCPP_RUN_VAR_COMPATIBILITY_TEST "Enable variable compatibility regression test" OFF) -option(CCPP_RUN_NESTED_SUITE_TEST "Enable nested suite regression test" OFF) - -message("") -message("OPENMP .............................. ${OPENMP}") -message("BUILD_SHARED_LIBS ................... ${BUILD_SHARED_LIBS}") -message("") -message("CCPP_FRAMEWORK_BUILD_DOCUMENTATION ...${CCPP_FRAMEWORK_BUILD_DOCUMENTATION}") -message("CCPP_FRAMEWORK_ENABLE_TESTS ......... ${CCPP_FRAMEWORK_ENABLE_TESTS}") -message("CCPP_RUN_ADVECTION_TEST ............. ${CCPP_RUN_ADVECTION_TEST}") -message("CCPP_RUN_CAPGEN_TEST ................ ${CCPP_RUN_CAPGEN_TEST}") -message("CCPP_RUN_DDT_HOST_TEST .............. ${CCPP_RUN_DDT_HOST_TEST}") -message("CCPP_RUN_VAR_COMPATIBILITY_TEST ..... ${CCPP_RUN_VAR_COMPATIBILITY_TEST}") -message("CCPP_RUN_NESTED_SUITE_TEST .......... ${CCPP_RUN_NESTED_SUITE_TEST}") -message("") - -set(CCPP_VERBOSITY "0" CACHE STRING "Verbosity level of output (default: 0)") - -# Warn user on conflicting test options -if(CCPP_RUN_ADVECTION_TEST OR - CCPP_RUN_CAPGEN_TEST OR - CCPP_RUN_DDT_HOST_TEST OR - CCPP_RUN_VAR_COMPATIBILITY_TEST) - set(CCPP_MANUALLY_DECLARED_TEST ON BOOL) -endif() -if(CCPP_MANUALLY_DECLARED_TEST AND CCPP_FRAMEWORK_ENABLE_TESTS) - message(WARNING "Detected a manual test flag and the flag to run all tests. If only expected to run a single test, please unset CCPP_FRAMEWORK_ENABLE_TESTS option.") -endif() -set(CCPP_RUNNING_TESTS CCPP_FRAMEWORK_ENABLE_TESTS OR CCPP_MANUALLY_DECLARED_TEST) - -# If running tests, set appropriate flags to help with debugging test issues. -if(CCPP_RUNNING_TESTS) - if(${CMAKE_Fortran_COMPILER_ID} STREQUAL "GNU") - ADD_COMPILE_OPTIONS(-fcheck=all) - ADD_COMPILE_OPTIONS(-fbacktrace) - ADD_COMPILE_OPTIONS(-ffpe-trap=zero) - ADD_COMPILE_OPTIONS(-finit-real=nan) - ADD_COMPILE_OPTIONS(-ggdb) - ADD_COMPILE_OPTIONS(-ffree-line-length-none) - ADD_COMPILE_OPTIONS(-cpp) - elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL "Intel") - ADD_COMPILE_OPTIONS(-fpe0) - ADD_COMPILE_OPTIONS(-warn) - ADD_COMPILE_OPTIONS(-traceback) - ADD_COMPILE_OPTIONS(-debug extended) - ADD_COMPILE_OPTIONS(-fpp) - ADD_COMPILE_OPTIONS(-diag-disable=10448) - elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL "IntelLLVM") - ADD_COMPILE_OPTIONS(-fpe0) - ADD_COMPILE_OPTIONS(-warn) - ADD_COMPILE_OPTIONS(-traceback) - ADD_COMPILE_OPTIONS(-debug full) - ADD_COMPILE_OPTIONS(-fpp) - elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "NVIDIA" OR ${CMAKE_Fortran_COMPILER_ID} STREQUAL "NVHPC") - ADD_COMPILE_OPTIONS(-Mnoipa) - ADD_COMPILE_OPTIONS(-traceback) - ADD_COMPILE_OPTIONS(-Mfree) - ADD_COMPILE_OPTIONS(-Ktrap=fp) - ADD_COMPILE_OPTIONS(-Mpreprocess) - else() - message (WARNING "This program may not be able to be compiled with compiler :${CMAKE_Fortran_COMPILER_ID}") - endif() -endif() - -# Use rpaths on MacOSX -set(CMAKE_MACOSX_RPATH 1) - -#------------------------------------------------------------------------------ -# Set MPI flags for Fortran with MPI F08 interface -find_package(MPI COMPONENTS Fortran REQUIRED) -if(NOT MPI_Fortran_HAVE_F08_MODULE) - message(FATAL_ERROR "MPI implementation does not support the Fortran 2008 mpi_f08 interface") -endif() - -#------------------------------------------------------------------------------ -# Set OpenMP flags for C/C++/Fortran -if(OPENMP) - find_package(OpenMP REQUIRED) -endif() - -#------------------------------------------------------------------------------ -# Set a default build type if none was specified -if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) - message(STATUS "Setting build type to 'Release' as none was specified.") - set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE) -endif() - -#------------------------------------------------------------------------------ -# Add the sub-directories -add_subdirectory(src) - -if(CCPP_RUNNING_TESTS) - enable_testing() - add_subdirectory(test) -endif() - -if (CCPP_FRAMEWORK_BUILD_DOCUMENTATION) - find_package(Doxygen REQUIRED) - add_subdirectory(doc) -endif() - diff --git a/LICENSE b/LICENSE index 996dc27b..add90622 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2017, NOAA, UCAR/NCAR CU/CIRES +Copyright 2026, NOAA, NAVY, UCAR/NCAR, CU/CIRES, CSU/CIRA Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 3447598e..11c71e52 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +TODO UPDATE ME + # CCPP Framework This repository contains the Common Community Physics Package (CCPP) Framework: The infrastructure that connects CCPP physics schemes with a host model, as well as stand-alone tools for use with CCPP. diff --git a/capgen/ccpp_capgen.py b/capgen/ccpp_capgen.py new file mode 100755 index 00000000..3f01918b --- /dev/null +++ b/capgen/ccpp_capgen.py @@ -0,0 +1,1267 @@ +#!/usr/bin/env python3 + +"""ccpp_capgen — next-generation CCPP cap code generator. + +This script replaces both ``ccpp_prebuild.py`` and ``ccpp_capgen.py`` from the +legacy toolchain. It reads host-model metadata files, scheme metadata files, +and suite XML definition files, resolves all variable connections, and writes: + +* ``ccpp_kinds.F90`` — kind parameter definitions +* ``_ccpp_cap.F90`` — static dispatch API (per-host; filename and module + name derived from ``--host-name``) +* ``ccpp__cap.F90`` — suite-level cap (state machine, group dispatch) +* ``ccpp___cap.F90`` — group-level cap (scheme call sites) +* ``ccpp__data.F90`` — suite-owned interstitial data module +* ``ccpp__types.F90`` — shared types (pointer wrappers, temp locals) +* ``ccpp_.meta`` — generated suite metadata (for inspection) +* ``datatable.xml`` — generator database for ``ccpp_datafile.py`` + +Usage +----- +:: + + ccpp_capgen.py \\ + --host-name \\ + --host-files \\ + --scheme-files \\ + --suites \\ + --output-root \\ + --kind-type NAME=[MODULE:]SPEC \\ # repeatable, see below + --verbose # once=INFO, twice=DEBUG + +``--kind-type`` +^^^^^^^^^^^^^^^ + +Each ``--kind-type`` entry maps a CCPP-visible kind name to a Fortran +precision constant. The syntax is:: + + --kind-type =[:] + +* ```` is the kind name as it will be published in ``ccpp_kinds`` and + referenced in scheme metadata (e.g. ``kind_phys``). +* ```` is the name of a precision constant (a kind parameter) defined + in some Fortran module. +* ```` is the Fortran module that defines ````. When + ``:`` is omitted, ```` must be a standard + ``ISO_FORTRAN_ENV`` constant (``REAL32``, ``REAL64``, ``INT32`` etc.) and + the module defaults to ``iso_fortran_env``. + +The flag may be specified multiple times. + +Examples:: + + --kind-type kind_phys=REAL64 + # → use iso_fortran_env, only: REAL64 + # integer, parameter, public :: kind_phys = REAL64 + + --kind-type kind_phys=my_host_kinds:kind_r8 + # → use my_host_kinds, only: kind_r8 + # integer, parameter, public :: kind_phys = kind_r8 + +If no ``--kind-type`` is supplied (or ``kind_phys`` is omitted from a +non-empty list), the generator injects ``kind_phys=iso_fortran_env:REAL64`` +and logs an INFO message. ``ccpp_kinds.F90`` is always written. + +Exit codes +---------- +0 — success +1 — user error (metadata problem, missing file, etc.) +2 — internal error (bug in the generator) +""" + +import argparse +import logging +import os +import sys +from typing import Dict, List, Optional, Tuple + +# Ensure the capgen package is importable when invoked directly. +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +_PACKAGE_DIR = os.path.dirname(_SCRIPT_DIR) +if _PACKAGE_DIR not in sys.path: + sys.path.insert(0, _PACKAGE_DIR) + +from metadata.parse_tools import CCPPError, init_log, set_log_level +from metadata.metadata_table import parse_metadata_file, MetadataTable +from metadata.variable_resolver import ( + build_ddt_module_map, + build_flat_host_dict, + SchemeStore, +) +from generator.kinds_writer import write_ccpp_kinds +from generator.suite_xml import parse_suite_xml_files +from generator.suite_resolver import ( + resolve_suite, iter_phase_calls, validate_init_dimensions, +) +from generator.group_cap import write_group_cap +from generator.suite_data import write_suite_data, write_suite_meta +from generator.suite_cap import write_suite_cap +from generator.suite_types import write_suite_types +from generator.host_cap import write_host_cap +from generator.host_constituents import write_host_constituents +from generator.datatable import write_datatable + + +######################################################################## +# Logging +######################################################################## + +_LOGGER = init_log('ccpp_capgen') + + +######################################################################## +# Framework-shipped metadata +######################################################################## + +# Path to the framework-shipped constituent module metadata, auto-included +# as a host metadata file so that the constituent DDT types are always known +# to the generator (even when the host metadata does not declare them). +_FRAMEWORK_SRC_DIR = os.path.join(_SCRIPT_DIR, 'src') +_FRAMEWORK_HOST_META = [ + os.path.join(_FRAMEWORK_SRC_DIR, 'ccpp_constituent_prop_mod.meta'), +] + +# Framework Fortran source files that must be compiled alongside the +# generated cap modules whenever any suite touches constituent state. +# Listed in datatable.xml's so host CMake projects pick them +# up via ccpp_datafile.py --utility-files / --ccpp-files queries. All +# of these live in :data:`_FRAMEWORK_SRC_DIR` (capgen's own ``src/``); +# capgen ships self-contained — no external src/ companion needed. +_FRAMEWORK_F90_FILES = [ + 'ccpp_constituent_prop_mod.F90', + 'ccpp_hashable.F90', + 'ccpp_hash_table.F90', + 'ccpp_scheme_utils.F90', +] + + +def _resolve_framework_f90_files() -> List[str]: + """Return absolute paths for the framework F90 files. + + Each name in :data:`_FRAMEWORK_F90_FILES` is looked up under + :data:`_FRAMEWORK_SRC_DIR` (``capgen/src/``). A missing file is + a hard error: capgen/src/ is the canonical (and only) location; + a missing file means the deployment is incomplete and the host + build would fail later with an opaque "Cannot open module file" + error. Surface it now with a precise message instead. + """ + found: List[str] = [] + missing: List[str] = [] + for name in _FRAMEWORK_F90_FILES: + p = os.path.join(_FRAMEWORK_SRC_DIR, name) + if os.path.isfile(p): + found.append(os.path.abspath(p)) + else: + missing.append(p) + if missing: + raise CCPPError( + "capgen deployment is incomplete: required framework " + "Fortran source file(s) not found under {!r}:\n {}\n" + "Vendor the missing file(s) into capgen/src/ (the " + "canonical location for files capgen emits a USE for).".format( + _FRAMEWORK_SRC_DIR, '\n '.join(missing), + ) + ) + return found + + +######################################################################## +# CLI +######################################################################## + +def _build_arg_parser() -> argparse.ArgumentParser: + """Build and return the argument parser. + + Returns + ------- + argparse.ArgumentParser + """ + parser = argparse.ArgumentParser( + prog='ccpp_capgen.py', + description='CCPP next-generation cap code generator', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument( + '--host-name', + required=True, + metavar='NAME', + help=( + 'Host model identifier. Drives the file and module name ' + 'of the generated static-API cap (``_ccpp_cap.F90`` / ' + '``module _ccpp_cap``) so multiple host integrations ' + 'can co-exist in one executable, and is written into ' + '``datatable.xml`` as the host var-dictionary name.' + ), + ) + parser.add_argument( + '--host-files', + required=True, + metavar='FILE[,FILE...]', + help='Comma-separated list of host-model metadata (.meta) files', + ) + parser.add_argument( + '--scheme-files', + required=True, + metavar='FILE[,FILE...]', + help='Comma-separated list of physics scheme metadata (.meta) files', + ) + parser.add_argument( + '--suites', + required=True, + metavar='FILE[,FILE...]', + help='Comma-separated list of suite XML definition (.xml) files', + ) + parser.add_argument( + '--output-root', + required=True, + metavar='DIR', + help='Output directory for all generated files', + ) + parser.add_argument( + '--kind-type', + action='append', + default=[], + metavar='NAME=[MODULE:]SPEC', + help=( + 'Map a CCPP kind name to a Fortran precision constant. Syntax: ' + '``=[:]``. When ``:`` is omitted, ' + '```` must be an ISO_FORTRAN_ENV constant (REAL32/REAL64/' + 'INT32/...) and the module defaults to ``iso_fortran_env``. ' + 'Examples: ``--kind-type kind_phys=REAL64``, ' + '``--kind-type kind_phys=my_host_kinds:kind_r8``. May be ' + 'specified multiple times. If kind_phys is not supplied, ' + '``kind_phys=iso_fortran_env:REAL64`` is injected automatically.' + ), + ) + parser.add_argument( + '--verbose', '-v', + action='count', + default=0, + help=( + 'Increase verbosity. Use once for INFO messages, ' + 'twice (-vv) for DEBUG messages.' + ), + ) + # legacy-compat: transient migration shim (delete the argument, + # the enable() call below, and the rest of the legacy_compat + # touchpoints when the migration is complete). + parser.add_argument( + '--legacy-mode', + action='store_true', + help=( + "TRANSIENT MIGRATION SHIM. Accept legacy CCPP standard " + "names (currently 'horizontal_loop_extent') in scheme " + "metadata and silently rewrite them to their canonical " + "capgen equivalents ('horizontal_dimension'). Emits a " + "loud warning at startup. Will be removed." + ), + ) + # dim-aliases: transient GFS-physics shim (delete the argument, + # the enable() call below, and the rest of the dim_aliases + # touchpoints when the workaround is removed). + parser.add_argument( + '--gfs-dim-aliases', + action='store_true', + help=( + "TRANSIENT GFS-PHYSICS SHIM. Treat a small audited list of " + "physically-equivalent vertical-axis standard names as the " + "same dimension during the host/scheme dim-position " + "identity check only (e.g. " + "'adjusted_vertical_layer_dimension_for_radiation' and " + "'vertical_composition_dimension' both compare equal to " + "'vertical_layer_dimension'). Variables keep their " + "original names everywhere else. Emits a loud warning at " + "startup. Will be removed." + ), + ) + # auto-clone-constituents: transient legacy shim (delete the + # argument, the enable() call below, and the rest of the + # auto_clone_constituents touchpoints when legacy hosts have + # migrated to explicit registration). + parser.add_argument( + '--legacy-auto-clone-constituents', + action='store_true', + help=( + "TRANSIENT LEGACY SHIM. Replicate original capgen's " + "auto-clone-static-constituent path: every is_constituent " + "scheme arg (advected / constituent / molar_mass) without " + "an explicit register-phase source is auto-registered into " + "the per-suite dynamic-constituents buffer using values " + "lifted directly from its scheme metadata. Accepts four " + "legacy attributes on scheme args (default_value, " + "min_value, water_species, mixing_ratio_type). " + "SINGLE-INSTANCE ONLY — the host must not declare the " + "(instance_number, number_of_instances) multi-instance " + "pair. Emits a loud warning at startup. Will be removed." + ), + ) + parser.add_argument( + '--no-host-introspection', + action='store_true', + help=( + "Stub the five suite-introspection routines in " + "_ccpp_cap.F90 (ccpp_physics_suite_list / " + "suite_part_list / suite_schemes / suite_variables / " + "suite_host_data). Signatures remain so callers still " + "link, but bodies set errflg=1 with a clear errmsg " + "(suite_list, which has no errflg, writes to error_unit " + "and returns an empty list). Use this to shrink the " + "generated _ccpp_cap.F90 from ~33000 lines to ~800 " + "for multi-suite builds where even -O1 cannot finish " + "compiling the introspection case-blocks." + ), + ) + parser.add_argument( + '--trace', + action='store_true', + help=( + "Set the default value of the per-module ``trace`` " + "parameter to ``.true.`` in every generated cap. The " + "gated ``if (trace) write(error_unit,*) ...`` lines are " + "ALWAYS emitted (one per cap subroutine that has " + "intent(in)/inout control dummies) so that strict " + "unused-variable warnings -- such as Intel oneAPI's -- " + "are silenced even when tracing is off. This flag only " + "flips the compile-time default; a developer can also " + "hand-edit ``logical, parameter :: trace`` in any " + "generated cap to ``.true.`` to enable tracing for that " + "module and rebuild." + ), + ) + return parser + + +# Standard ISO_FORTRAN_ENV kind constants accepted as a bare ```` (i.e. +# without an explicit ``:`` prefix). Compared case-insensitively. +_ISO_FORTRAN_KINDS = frozenset({ + 'INT8', 'INT16', 'INT32', 'INT64', + 'REAL32', 'REAL64', 'REAL128', +}) + +_ISO_FORTRAN_MODULE = 'iso_fortran_env' + + +def _parse_kind_types( + kind_type_args: List[str], +) -> Dict[str, Tuple[str, str]]: + """Parse ``--kind-type NAME=[MODULE:]SPEC`` arguments into a mapping. + + Parameters + ---------- + kind_type_args : list of str + Each entry must have the form ``=[:]``. + + Returns + ------- + dict + Mapping from kind name to a ``(module, spec)`` tuple. + + Raises + ------ + CCPPError + If any entry is malformed, has a duplicate name, or omits the module + for a non-ISO ````. + + Examples + -------- + Default ``iso_fortran_env`` module when spec is a known ISO kind: + + >>> _parse_kind_types(['kind_phys=REAL64', 'kind_dyn=REAL32']) + {'kind_phys': ('iso_fortran_env', 'REAL64'), 'kind_dyn': ('iso_fortran_env', 'REAL32')} + + Explicit host-supplied module: + + >>> _parse_kind_types(['kind_phys=my_host_kinds:kind_r8']) + {'kind_phys': ('my_host_kinds', 'kind_r8')} + + Mixed: + + >>> sorted(_parse_kind_types([ + ... 'kind_iso=REAL64', + ... 'kind_host=my_kinds:kind_r4', + ... ]).items()) + [('kind_host', ('my_kinds', 'kind_r4')), ('kind_iso', ('iso_fortran_env', 'REAL64'))] + + Malformed (missing ``=``): + + >>> _parse_kind_types(['bad_entry']) + Traceback (most recent call last): + ... + metadata.parse_tools.parse_source.CCPPError: --kind-type 'bad_entry' must have the form NAME=[MODULE:]SPEC + + Duplicate entry: + + >>> _parse_kind_types(['kind_phys=REAL64', 'kind_phys=REAL32']) + Traceback (most recent call last): + ... + metadata.parse_tools.parse_source.CCPPError: Duplicate --kind-type entry for 'kind_phys' + + Non-ISO spec without explicit module: + + >>> _parse_kind_types(['kind_phys=kind_r8']) + Traceback (most recent call last): + ... + metadata.parse_tools.parse_source.CCPPError: --kind-type 'kind_phys=kind_r8': spec 'kind_r8' is not a standard ISO_FORTRAN_ENV constant; supply the module explicitly as : + """ + mapping: Dict[str, Tuple[str, str]] = {} + for entry in kind_type_args: + head, sep, tail = entry.partition('=') + if not sep or not head.strip() or not tail.strip(): + raise CCPPError( + "--kind-type '{}' must have the form NAME=[MODULE:]SPEC".format(entry) + ) + kind_name = head.strip() + rhs = tail.strip() + + # Split the right-hand side on ':'. At most one colon is permitted. + rhs_parts = rhs.split(':') + if len(rhs_parts) == 1: + spec = rhs_parts[0].strip() + module = _ISO_FORTRAN_MODULE + if spec.upper() not in _ISO_FORTRAN_KINDS: + raise CCPPError( + "--kind-type '{}': spec '{}' is not a standard " + "ISO_FORTRAN_ENV constant; supply the module " + "explicitly as :".format(entry, spec) + ) + elif len(rhs_parts) == 2: + module = rhs_parts[0].strip() + spec = rhs_parts[1].strip() + if not module or not spec: + raise CCPPError( + "--kind-type '{}': both and must be " + "non-empty when using the : form".format(entry) + ) + else: + raise CCPPError( + "--kind-type '{}': at most one ':' is permitted " + "(syntax is NAME=[MODULE:]SPEC)".format(entry) + ) + + if kind_name in mapping: + raise CCPPError( + "Duplicate --kind-type entry for '{}'".format(kind_name) + ) + mapping[kind_name] = (module, spec) + return mapping + + +def _ensure_kind_phys_default( + kind_types: Dict[str, Tuple[str, str]], + log: logging.Logger, +) -> Dict[str, Tuple[str, str]]: + """Inject ``kind_phys=iso_fortran_env:REAL64`` if not already mapped. + + Mutates and returns *kind_types*. Logs an INFO message when the default + is injected, so users always know that the fallback is in effect. + """ + if 'kind_phys' not in kind_types: + kind_types['kind_phys'] = (_ISO_FORTRAN_MODULE, 'REAL64') + log.info( + "kind_phys not supplied via --kind-type or metadata kind_spec; " + "defaulting to REAL64 from iso_fortran_env" + ) + return kind_types + + +def _collect_metadata_kind_specs( + tables: List[MetadataTable], +) -> Dict[str, Tuple[str, str]]: + """Aggregate ``kind_spec`` declarations across loaded metadata tables. + + Each table contributes zero or more ``(kind_name, module, spec)`` triples + via :attr:`MetadataTable.kind_specs`. All contributions for the same + ``kind_name`` must agree; identical duplicates are collapsed silently + while a divergent ``(module, spec)`` raises :exc:`CCPPError` with a + message naming both source files. + + Parameters + ---------- + tables : list of MetadataTable + Host and scheme metadata tables, in any order. + + Returns + ------- + dict + Mapping ``kind_name -> (module, spec)``. + + Raises + ------ + CCPPError + If two tables declare the same ``kind_name`` with different + ``(module, spec)`` pairs. + """ + result: Dict[str, Tuple[str, str]] = {} + sources: Dict[str, str] = {} + for tbl in tables: + for kind_name, module, spec in tbl.kind_specs: + pair = (module, spec) + origin = "{} (table '{}')".format(tbl.file_path, tbl.table_name) + existing = result.get(kind_name) + if existing is None: + result[kind_name] = pair + sources[kind_name] = origin + elif existing != pair: + raise CCPPError( + "Conflicting kind_spec for kind '{}': {} declares " + "'{}:{}' but {} declares '{}:{}'".format( + kind_name, sources[kind_name], + existing[0], existing[1], + origin, pair[0], pair[1], + ) + ) + return result + + +def _merge_cli_and_metadata_kinds( + cli_kinds: Dict[str, Tuple[str, str]], + meta_kinds: Dict[str, Tuple[str, str]], +) -> Dict[str, Tuple[str, str]]: + """Combine ``--kind-type`` CLI mappings with metadata-declared kinds. + + For any ``kind_name`` defined in both sides the ``(module, spec)`` pair + must match exactly. Identical pairs collapse silently; mismatches raise + :exc:`CCPPError`. + + Parameters + ---------- + cli_kinds : dict + Mapping from :func:`_parse_kind_types`. + meta_kinds : dict + Mapping from :func:`_collect_metadata_kind_specs`. + + Returns + ------- + dict + Merged mapping ``kind_name -> (module, spec)``. + + Raises + ------ + CCPPError + If CLI and metadata declare the same kind name with different + ``(module, spec)`` pairs. + """ + merged = dict(cli_kinds) + for kind_name, pair in meta_kinds.items(): + existing = merged.get(kind_name) + if existing is None: + merged[kind_name] = pair + elif existing != pair: + raise CCPPError( + "Kind '{}' declared inconsistently: --kind-type says " + "'{}:{}' but metadata kind_spec says '{}:{}'".format( + kind_name, existing[0], existing[1], pair[0], pair[1], + ) + ) + return merged + + +def _split_file_list(arg: str) -> List[str]: + """Split a comma-separated file-list argument, stripping whitespace. + + Parameters + ---------- + arg : str + Comma-separated list of file paths. + + Returns + ------- + list of str + + Examples + -------- + >>> _split_file_list('a.meta, b.meta, c.meta') + ['a.meta', 'b.meta', 'c.meta'] + >>> _split_file_list('single.meta') + ['single.meta'] + >>> _split_file_list('') + [] + """ + return [f.strip() for f in arg.split(',') if f.strip()] + + +######################################################################## +# Metadata loading +######################################################################## + +# Loop-bound standard names that must never appear as variable dimensions. +# These are control variables (scalars passed as subroutine arguments) and +# using them as array dimensions indicates a porting error from the legacy +# toolchain. Remove this guard once migration is complete. +_FORBIDDEN_DIMENSION_NAMES = frozenset({ + 'horizontal_loop_extent', + 'horizontal_loop_begin', + 'horizontal_loop_end', +}) + + +def _check_no_loop_dimensions(tables: list) -> None: + """Raise CCPPError if any variable uses a forbidden dimension name. + + Parameters + ---------- + tables : list of MetadataTable + + Raises + ------ + CCPPError + If any variable's dimensions list contains a name from + ``_FORBIDDEN_DIMENSION_NAMES``. All violations are collected and + reported together. + """ + errors = [] + for tbl in tables: + for sec in tbl.sections(): + for var in sec.variables: + for dim in var.dimensions: + if dim in _FORBIDDEN_DIMENSION_NAMES: + errors.append( + "Variable '{}' (standard_name='{}') in table " + "'{}' (type={}) in '{}' uses '{}' as a " + "dimension. Loop-bound control/legacy vars must " + "not appear in dimension attributes; use " + "horizontal_dimension instead.".format( + var.local_name, var.standard_name, + tbl.table_name, tbl.table_type, + tbl.file_path, dim, + ) + ) + if errors: + raise CCPPError( + "Forbidden dimension names found in metadata:\n\n{}".format( + '\n\n'.join("ERROR: " + e for e in errors) + ) + ) + +def _load_metadata_files( + file_list: List[str], + expected_types: frozenset, + label: str, +) -> List[MetadataTable]: + """Load and validate a list of metadata files. + + Parameters + ---------- + file_list : list of str + Paths to ``.meta`` files. + expected_types : frozenset of str + Table types that are acceptable in these files. Any table with a + different type raises a :exc:`CCPPError`. + label : str + Human-readable description (``'host'`` or ``'scheme'``) used in + error messages. + + Returns + ------- + list of MetadataTable + All tables parsed from all files, in order. + + Raises + ------ + CCPPError + On any parse error or unexpected table type. + """ + tables: List[MetadataTable] = [] + for fpath in file_list: + _LOGGER.info("Reading %s metadata: %s", label, fpath) + file_tables = parse_metadata_file(fpath) + for tbl in file_tables: + if tbl.table_type not in expected_types: + raise CCPPError( + "Unexpected table type '{}' in {} metadata file '{}'; " + "expected one of {}".format( + tbl.table_type, label, fpath, sorted(expected_types) + ) + ) + _check_no_loop_dimensions(file_tables) + tables.extend(file_tables) + return tables + + +######################################################################## +# Control-variable validation +######################################################################## + +# Required control variables: (standard_name, expected_fortran_type, description) +_REQUIRED_CTRL_VARS = [ + ('suite_name', 'character', 'drives suite dispatch'), + ('group_name', 'character', 'drives per-group dispatch inside ccpp_physics_* (each suite_cap emits a select case on this name)'), + ('horizontal_loop_begin', 'integer', 'lower horizontal slice bound at scheme call sites'), + ('horizontal_loop_end', 'integer', 'upper horizontal slice bound at scheme call sites'), + ('number_of_physics_threads','integer', 'physics-internal thread budget (pass 1 if unused)'), + ('ccpp_error_code', 'integer', 'CCPP error code'), + ('ccpp_error_message', 'character', 'CCPP error message'), +] +# NOTE: the threading index/count (``thread_number`` / ``number_of_threads``) +# is NOT required — it is a paired-optional control pair, fully symmetric with +# (``instance_number`` / ``number_of_instances``); see +# ``_PAIRED_OPTIONAL_CTRL_VARS`` below. ``number_of_physics_threads`` is a +# separate, unpaired scheme-facing scalar that stays unconditionally required. + +# Paired-optional control variables. Each entry is an (index, count) pair: +# the host declares BOTH members (in ``type=control``) or NEITHER; declaring +# exactly one is a hard error. Declaring a pair opts the host into that +# multi- API — the index flows as a per-call control dummy and the count +# gives the bound. When a pair is absent the public API drops both args and +# the framework uses literal ``1`` wherever the index would appear. A host +# variable may be dimensioned by the count standard name only when its pair is +# declared (otherwise the resolver's scalar-index collapse raises — it needs +# the index variable in scope). +# +# The two pairs are fully symmetric (decision 2026-06-09): +# * (instance_number, number_of_instances) — multi-instance API. The +# framework reads ``number_of_instances`` at register/init to size its +# own per-instance state (``ccpp_suite_data(:)``, ``ccpp_group_state(:)``). +# * (thread_number, number_of_threads) — multi-threading API. +# ``thread_number`` indexes host-owned per-thread containers; +# ``number_of_threads`` is carried as a control dummy (the framework owns +# no per-thread state yet, so its value is not consumed — kept for symmetry +# with ``number_of_instances`` and future per-thread sizing). +# (A chunk/block index is intentionally NOT a control pair: capgen's +# slice-based design passes the current chunk as a horizontal range via +# horizontal_loop_begin/end, so no scheme ever indexes by chunk inside a call.) +# Each entry: (index std_name, count std_name, index description, count description). +_PAIRED_OPTIONAL_CTRL_VARS = [ + ('instance_number', 'number_of_instances', + 'current model instance index', 'total number of model instances'), + ('thread_number', 'number_of_threads', + 'current thread index', 'total thread count'), +] + + +def _validate_required_control_vars( + host_name: str, + host_dict: dict, +) -> None: + """Check that every required control variable is present in *host_dict*. + + Collects all failures and raises a single :exc:`CCPPError` listing them. + + Parameters + ---------- + host_name : str + Host model identifier, used in error messages so the developer + can tell which host the failure refers to when more than one + capgen invocation is in flight. + host_dict : dict + Flat host variable dictionary built by :func:`build_flat_host_dict`. + + Raises + ------ + CCPPError + If any required control variable is missing, not marked as a control + variable, has the wrong Fortran type, or is not a scalar. + """ + errors = [] + + def _check_control_var(std_name, expected_type, description, required: bool) -> None: + """Validate a variable that must live in a ``type=control`` table.""" + entry = host_dict.get(std_name) + + if entry is None: + if required: + errors.append( + "Required control variable '{}' not found in host '{}' " + "type=control metadata.\n" + " This variable {}. Add it to a " + "[ccpp-table-properties] / type=control block in the " + "host metadata files.".format(std_name, host_name, description) + ) + return + + if not entry.is_control: + errors.append( + "Variable '{}' must be declared in a type=control table " + "for host '{}', but it was found in a type=host table.\n" + " Move it to a [ccpp-table-properties] / type=control " + "block.".format(std_name, host_name) + ) + return + + if entry.type.lower() != expected_type.lower(): + errors.append( + "Required control variable '{}' in host '{}' has Fortran " + "type '{}' but '{}' is required.".format( + std_name, host_name, entry.type, expected_type + ) + ) + + if entry.dimensions: + errors.append( + "Required control variable '{}' in host '{}' must be a " + "scalar (rank-0) but has dimensions {}.".format( + std_name, host_name, entry.dimensions + ) + ) + + for std_name, expected_type, description in _REQUIRED_CTRL_VARS: + _check_control_var(std_name, expected_type, description, required=True) + + # Paired-optional control pairs (see _PAIRED_OPTIONAL_CTRL_VARS): for each + # (index, count) pair the host declares both members in a type=control + # table or neither. Declaring exactly one is an error. Both pairs — + # (instance_number, number_of_instances) and (thread_number, + # number_of_threads) — are validated identically. + for idx_name, cnt_name, idx_desc, cnt_desc in _PAIRED_OPTIONAL_CTRL_VARS: + _check_control_var(idx_name, 'integer', idx_desc, required=False) + _check_control_var(cnt_name, 'integer', cnt_desc, required=False) + + idx_present = host_dict.get(idx_name) is not None + cnt_present = host_dict.get(cnt_name) is not None + if idx_present ^ cnt_present: + present, missing = ( + (idx_name, cnt_name) if idx_present else (cnt_name, idx_name) + ) + errors.append( + "Host '{}' declares '{}' in a type=control table but is " + "missing its paired variable '{}' (which must also be in a " + "type=control table).\n" + " '{}' and '{}' are a paired-optional control pair: declare " + "both members to opt into that API, or neither.".format( + host_name, present, missing, idx_name, cnt_name, + ) + ) + + # Control-table allowlist: a type=control table may declare ONLY the + # framework's known control variables — the unconditionally required set + # plus the members of the paired-optional pairs. Anything else in a + # type=control table is a hard error. Host-specific quantities that + # schemes consume belong in a type=host table; the subcycle loop variables + # (ccpp_loop_counter / ccpp_loop_extent) are generator-owned locals the + # host never declares. + allowed_control = {name for name, _type, _desc in _REQUIRED_CTRL_VARS} + for idx_name, cnt_name, _idesc, _cdesc in _PAIRED_OPTIONAL_CTRL_VARS: + allowed_control.add(idx_name) + allowed_control.add(cnt_name) + for std_name, entry in sorted(host_dict.items()): + if entry.is_control and std_name not in allowed_control: + errors.append( + "Variable '{}' is declared in a type=control table for host " + "'{}' but is not a recognized framework control variable.\n" + " A type=control table may declare only: {}.\n" + " If '{}' is a host quantity that schemes consume, declare it " + "in a type=host table instead.".format( + std_name, host_name, ', '.join(sorted(allowed_control)), + std_name, + ) + ) + + if errors: + raise CCPPError( + "Host '{}' has invalid control-variable metadata:\n\n{}".format( + host_name, + '\n\n'.join("ERROR: " + e for e in errors), + ) + ) + + +######################################################################## +# Entry point +######################################################################## + +def capgen( + host_name: str, + host_files: List[str], + scheme_files: List[str], + suite_files: List[str], + output_root: str, + kind_types: Dict[str, Tuple[str, str]], + logger: Optional[logging.Logger] = None, + no_host_introspection: bool = False, + trace: bool = False, + return_state: bool = False, +): + """Programmatic entry point for the cap generator. + + Mirrors the CLI behaviour. Both the CLI and programmatic paths call + this function. + + Parameters + ---------- + host_name : str + Host model identifier. Drives the file and module name of the + generated static-API cap (``_ccpp_cap.F90`` / ``module + _ccpp_cap``) and is written into ``datatable.xml``. + host_files : list of str + Host metadata (``.meta``) file paths. + scheme_files : list of str + Scheme metadata (``.meta``) file paths. + suite_files : list of str + Suite XML (``.xml``) file paths. + output_root : str + Directory where all generated files are written. + kind_types : dict + Mapping ``kind_name -> (module_name, kind_spec)``. May be empty; + ``kind_phys=(iso_fortran_env, REAL64)`` is injected automatically + when missing. + logger : logging.Logger, optional + Logger to use. Defaults to the module-level logger. + + Raises + ------ + CCPPError + On any user-facing error. + """ + log = logger or _LOGGER + + # Snapshot the CLI-provided kinds; the default ``kind_phys`` and any + # metadata-declared kind_specs are folded in below, after metadata loads. + cli_kind_types = dict(kind_types) + + # ---- validate output directory ----------------------------------------- + os.makedirs(output_root, exist_ok=True) + + # ---- load host metadata (host + control tables) ------------------------- + log.info("Loading host metadata for host '%s'", host_name) + framework_meta = [p for p in _FRAMEWORK_HOST_META if os.path.isfile(p)] + if framework_meta: + log.info("Auto-including framework metadata: %s", framework_meta) + host_tables = _load_metadata_files( + framework_meta + list(host_files), + expected_types=frozenset({'host', 'control', 'ddt'}), + label='host', + ) + log.info("Loaded %d host/control/ddt tables", len(host_tables)) + + # ---- load scheme metadata ----------------------------------------------- + log.info("Loading scheme metadata") + scheme_tables = _load_metadata_files( + scheme_files, + expected_types=frozenset({'scheme', 'ddt'}), + label='scheme', + ) + log.info("Loaded %d scheme/ddt tables", len(scheme_tables)) + + # ---- merge --kind-type with metadata kind_spec declarations ----------- + meta_kind_types = _collect_metadata_kind_specs(host_tables + scheme_tables) + if meta_kind_types: + log.info( + "Found %d kind_spec declaration(s) in metadata: %s", + len(meta_kind_types), sorted(meta_kind_types), + ) + kind_types = _merge_cli_and_metadata_kinds(cli_kind_types, meta_kind_types) + kind_types = _ensure_kind_phys_default(kind_types, log) + + # ---- build flat host variable dictionary -------------------------------- + host_only = [t for t in host_tables if t.table_type == 'host'] + control_only = [t for t in host_tables if t.table_type == 'control'] + ddt_from_host = [t for t in host_tables if t.table_type == 'ddt'] + ddt_from_schemes = [t for t in scheme_tables if t.table_type == 'ddt'] + all_ddt_tables = ddt_from_host + ddt_from_schemes + + host_dict = build_flat_host_dict(host_only, control_only, all_ddt_tables) + log.info("Host dictionary contains %d variables", len(host_dict)) + + # Map DDT type name → defining Fortran module, derived from co-located + # tables in each .meta file. Used by the suite data generator to emit + # USE statements for DDT-typed suite-owned variables. + ddt_module_map = build_ddt_module_map(host_tables + scheme_tables) + + # ---- Phase 1 validation: required control variables --------------------- + _validate_required_control_vars(host_name, host_dict) + + # auto-clone-constituents: enforce the single-instance constraint + # of the transient legacy shim. No-op when the shim is disabled. + from metadata import auto_clone_constituents + auto_clone_constituents.require_single_instance_host(host_dict) + + # Signal which instance API the host opted into so users can tell which + # branch the generator took. Paired-presence has already been enforced. + if host_dict.get('instance_number') is not None: + log.info("Host '%s' declares instance_number — generating " + "multi-instance API.", host_name) + else: + log.info("Host '%s' did not declare instance_number — generating " + "single-instance API (per-instance arrays sized to 1).", + host_name) + + # ---- build scheme metadata store ---------------------------------------- + scheme_store = SchemeStore.build_from(scheme_tables) + log.info("Scheme store contains %d schemes: %s", + len(scheme_store.scheme_names()), scheme_store.scheme_names()) + + # ---- write ccpp_kinds.F90 (always generated) --------------------------- + # Every writer below logs its own "Wrote " / "Unchanged: " + # line via the write-if-changed helper when *logger* is threaded + # through. Don't duplicate that log here. + kinds_path = write_ccpp_kinds(kind_types, output_root, logger=log) + + # ---- parse suite XML files ---------------------------------------------- + suites = parse_suite_xml_files(suite_files, output_root, log) + log.info("Loaded %d suite(s): %s", len(suites), [s.name for s in suites]) + + # ---- resolve and generate per-suite outputs ---------------------------- + suite_names = [] + suite_resolutions = [] + + for suite in suites: + log.info("Resolving suite '%s'", suite.name) + suite_res = resolve_suite(suite, scheme_store, host_dict) + # Fail early (before any cap is written) if a non-allocatable + # suite-owned variable is dimensioned by a scheme-updated quantity + # that isn't known when suite_data_init_fields allocates it. + validate_init_dimensions(suite_res) + suite_names.append(suite.name) + suite_resolutions.append(suite_res) + + # Group caps + for resolved_group in suite_res.groups: + write_group_cap( + suite.name, resolved_group.group_name, resolved_group, host_dict, output_root, + logger=log, + trace=trace, + ) + + # Suite data module + write_suite_data( + suite.name, suite_res.suite_vars, output_root, host_dict, + ddt_module_map=ddt_module_map, logger=log, + ) + + # Suite metadata (for inspection) + write_suite_meta( + suite.name, suite_res.suite_vars, output_root, logger=log, + ) + + # Suite types module (only when optional args are present) + write_suite_types( + suite.name, suite_res, output_root, + ddt_module_map=ddt_module_map, logger=log, + ) + + # Suite cap + write_suite_cap( + suite.name, suite_res, scheme_store, output_root, host_dict, + logger=log, + trace=trace, + ) + + # ---- host cap (one file for all suites) -------------------------------- + write_host_cap( + host_name, suite_names, suite_resolutions, output_root, + host_dict, scheme_store, + logger=log, + no_host_introspection=no_host_introspection, + trace=trace, + ) + + # ---- host-wide constituent module (only when any suite touches + # constituent state) ------------------------------------------------ + host_consts_path = write_host_constituents( + suite_resolutions, output_root, host_dict=host_dict, logger=log, + ) + + # ---- datatable.xml ------------------------------------------------------ + abs_root = os.path.abspath(output_root) + utility_paths = [ + os.path.join(abs_root, 'ccpp_kinds.F90'), + ] + if host_consts_path: + utility_paths.append(host_consts_path) + # The generated ccpp_host_constituents.F90 USEs ccpp_constituent_prop_mod + # (and transitively ccpp_hashable / ccpp_hash_table); host code that + # calls ccpp_constituent_index pulls in ccpp_scheme_utils. Add all + # framework F90 dependencies so the host build picks them up. + utility_paths.extend(_resolve_framework_f90_files()) + host_file_paths = [ + os.path.join(abs_root, '{}_ccpp_cap.F90'.format(host_name)), + ] + suite_file_paths = [] + suite_meta_paths = [] + for sname, suite_resolution in zip(suite_names, suite_resolutions): + suite_file_paths.append( + os.path.join(abs_root, 'ccpp_{}_cap.F90'.format(sname)) + ) + suite_file_paths.append( + os.path.join(abs_root, 'ccpp_{}_data.F90'.format(sname)) + ) + # Types module is only present when optional args exist. + types_file = os.path.join(abs_root, 'ccpp_{}_types.F90'.format(sname)) + if os.path.isfile(types_file): + suite_file_paths.append(types_file) + for resolved_group in suite_resolution.groups: + suite_file_paths.append( + os.path.join( + abs_root, + 'ccpp_{}_{}_cap.F90'.format(sname, resolved_group.group_name), + ) + ) + suite_meta_paths.append( + os.path.join(abs_root, 'ccpp_{}_data.meta'.format(sname)) + ) + # Expanded SDFs (one per parsed suite) are inspection artifacts; carry + # the paths set by parse_suite_xml() forward into datatable.xml. + expanded_sdf_paths = [s.expanded_file for s in suites if s.expanded_file] + # Collect dependency paths. Host/control/ddt tables always contribute + # (their Fortran is shared across suites). Scheme-type tables only + # contribute when the scheme is actually referenced by a resolved + # suite — group phase calls, the suite-level scheme, or the + # suite-level scheme. DDT tables co-located in a scheme + # ``.meta`` file (``type = ddt`` block alongside ``type = scheme`` + # blocks) always contribute since DDT modules are shared host-side + # data, not gated on which scheme uses them. Unreferenced scheme + # metadata may sit on the CLI line for build-system convenience; we + # don't want its dependencies to leak into datatable.xml. Duplicates + # are collapsed by ``write_datatable``. + used_scheme_names: set = set() + for suite_resolution in suite_resolutions: + for resolved_group in suite_resolution.groups: + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + used_scheme_names.add(resolved_call.scheme_name) + if suite_resolution.suite_init_call is not None: + used_scheme_names.add(suite_resolution.suite_init_call.scheme_name) + if suite_resolution.suite_final_call is not None: + used_scheme_names.add(suite_resolution.suite_final_call.scheme_name) + + dependency_paths = [] + for tbl in host_tables: + dependency_paths.extend(tbl.dependencies) + for tbl in scheme_tables: + if tbl.table_type != 'scheme': + # DDT (or other non-scheme) tables that live alongside scheme + # tables in scheme metadata files — always contribute. + dependency_paths.extend(tbl.dependencies) + elif tbl.table_name in used_scheme_names: + dependency_paths.extend(tbl.dependencies) + + # Used-scheme Fortran source paths. Convention (shared with the + # validator's ``_fortran_file_for_table``): the ``.F90`` (or ``.F`` / + # ``.f90`` / ``.f``) lives under ``table.source_path`` with the same + # base name as the ``.meta`` file. Multiple scheme tables in one + # ``.meta`` share that single source file, so dedupe per path. + scheme_file_paths: List[str] = [] + _seen_scheme_files: set = set() + for tbl in scheme_tables: + # DDT tables inside scheme .meta files do not correspond to a + # scheme .F90 — skip them outright. + if tbl.table_type != 'scheme': + continue + if tbl.table_name not in used_scheme_names: + continue + meta_base = os.path.splitext(os.path.basename(tbl.file_path))[0] + search_dir = tbl.source_path or os.path.dirname( + os.path.abspath(tbl.file_path) + ) + resolved = None + for ext in ('.F90', '.f90', '.F', '.f'): + candidate = os.path.join(search_dir, meta_base + ext) + if os.path.isfile(candidate): + resolved = candidate + break + if resolved is None: + # Fall back to the canonical .F90 guess so the build-system + # query returns a useful (if missing) path; surface the gap + # at the same time so the user can fix source_path. + resolved = os.path.join(search_dir, meta_base + '.F90') + log.warning( + "Scheme '%s': no Fortran source found under '%s' for " + "any of .F90/.f90/.F/.f; using '%s' as the datatable " + "entry. Check the scheme's source_path table-property.", + tbl.table_name, search_dir, resolved, + ) + if resolved not in _seen_scheme_files: + _seen_scheme_files.add(resolved) + scheme_file_paths.append(resolved) + + write_datatable( + suite_resolutions, scheme_store, utility_paths, suite_file_paths, + output_root, host_file_paths=host_file_paths, + scheme_file_paths=scheme_file_paths, + dependency_paths=dependency_paths, + suite_meta_paths=suite_meta_paths, + expanded_sdf_paths=expanded_sdf_paths, + host_dict=host_dict, host_name=host_name, + logger=log, + ) + + log.info("Cap generation complete.") + + # When *return_state* is requested, hand the resolved state back + # to the caller so external tools (host-side compat adapters, + # debug utilities, downstream code generators) can consume the + # in-memory ``host_dict`` and ``suite_resolutions`` without + # re-running the load + resolve passes. Returns ``None`` + # otherwise: the canonical signature is "side effects only". + if return_state: + return host_dict, suite_resolutions + return None + + +def main(argv: Optional[List[str]] = None) -> int: + """Command-line entry point. + + Parameters + ---------- + argv : list of str, optional + Override ``sys.argv[1:]`` (used by tests). + + Returns + ------- + int + Exit code: 0 = success, 1 = user error, 2 = internal error. + """ + parser = _build_arg_parser() + args = parser.parse_args(argv) + + # ---- configure logging ------------------------------------------------- + if args.verbose == 0: + set_log_level(_LOGGER, logging.WARNING) + elif args.verbose == 1: + set_log_level(_LOGGER, logging.INFO) + else: + set_log_level(_LOGGER, logging.DEBUG) + + # legacy-compat: transient migration shim. Emit the loud banner + # before any parsing happens so user has fair warning. + if args.legacy_mode: + from metadata import legacy_compat + legacy_compat.enable(_LOGGER) + + # dim-aliases: transient GFS-physics shim. Emit the loud banner + # before any parsing happens so user has fair warning. + if args.gfs_dim_aliases: + from metadata import dim_aliases + dim_aliases.enable(_LOGGER) + + # auto-clone-constituents: transient legacy shim. Emit the loud + # banner before any parsing happens so the user has fair warning. + # The single-instance assertion (host MUST NOT declare the + # instance_number / number_of_instances pair) runs later, after + # host metadata has been parsed, in ``capgen()``. + if args.legacy_auto_clone_constituents: + from metadata import auto_clone_constituents + auto_clone_constituents.enable(_LOGGER) + + # ---- parse kind types -------------------------------------------------- + try: + kind_types = _parse_kind_types(args.kind_type) + except CCPPError as exc: + _LOGGER.error("%s", exc) + return 1 + + # ---- call the generator ------------------------------------------------ + try: + capgen( + host_name=args.host_name, + host_files=_split_file_list(args.host_files), + scheme_files=_split_file_list(args.scheme_files), + suite_files=_split_file_list(args.suites), + output_root=args.output_root, + kind_types=kind_types, + no_host_introspection=args.no_host_introspection, + trace=args.trace, + ) + except CCPPError as exc: + _LOGGER.error("%s", exc) + return 1 + except Exception as exc: # pylint: disable=broad-except + _LOGGER.error("Internal error: %s", exc, exc_info=True) + return 2 + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/ccpp_datafile.py b/capgen/ccpp_datafile.py similarity index 50% rename from scripts/ccpp_datafile.py rename to capgen/ccpp_datafile.py index 2c546528..4ebfcd62 100755 --- a/scripts/ccpp_datafile.py +++ b/capgen/ccpp_datafile.py @@ -1,98 +1,112 @@ #!/usr/bin/env python3 -"""Code to generate and query the CCPP datafile returned by capgen. -The CCPP datafile is a database consisting of several tables: -- A list of all generated files, broken into groups for host cap, - suite caps, and ccpp_kinds. -- A list of scheme entries, keyed by scheme name -- A list of CCPP metadata files actually used by capgen, broken into groups - for host-model metadata and scheme metadata. These filenames may serve - as keys -- A list of variable entries, keyed by standard name. +"""Query CLI for the ``datatable.xml`` produced by ``ccpp_capgen.py``. + +This is the read-side companion to :mod:`generator.datatable`. The writer +half lives in the generator package; this module is a pure-Python, +dependency-free reader (only the stdlib ``xml.etree.ElementTree``) so it +can be invoked from CMake or any other build-time tooling. + +The flag surface mirrors the original ``scripts/ccpp_datafile.py``: + + * Fortran-source enumeration: + ``--host-files`` / ``--suite-files`` / ``--utility-files`` / + ``--capgen-files``. All four return *only* generated Fortran files + (``.F90``). ``--capgen-files`` is the union of the other three. + * Non-Fortran enumeration: + ``--inspection-files`` — generated ``.meta`` and expanded SDF XML. + * ``--process-list`` / ``--module-list`` / ``--dependencies`` + * ``--suite-list`` + * ``--required-variables`` / ``--input-variables`` / + ``--output-variables`` / ``--host-variables`` + * ``--show`` (pretty-print) + * ``--separator``, ``--exclude-protected``, ``--line-wrap``, ``--indent`` + +Exactly one report action is required per invocation. + +Notes specific to capgen +--------------------------- +* ``--host-files`` returns ``_ccpp_cap.F90`` (the per-host static API; + filename and module name derived from ``--host-name`` at generation time). +* ``--capgen-files`` enumerates Fortran sources only. Non-Fortran inspection + artifacts (``ccpp_.meta``, ``ccpp__expanded.xml``) are + reported via ``--inspection-files``. +* ``--process-list`` is supported syntactically but will return an empty + string: capgen does not currently record a ``process`` attribute on + scheme entries. """ -## NB: A new report must be added in two places: -## 1) In the list of DatatableReport._valid_reports -## 2) As an option in datatable_report - -# Python library imports import argparse -import logging -import os import sys import xml.etree.ElementTree as ET -# CCPP framework imports -from framework_env import CCPPFrameworkEnv -from metadata_table import UNKNOWN_PROCESS_TYPE -from metavar import Var -from parse_tools import read_xml_file, write_xml_file -from parse_tools import ParseContext, ParseSource -from suite_objects import Subcycle - -# Global data +from typing import List, Optional + _INDENT_STR = " " -# Used for creating template variables -_MVAR_DUMMY_RUN_ENV = CCPPFrameworkEnv(None, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - -## datatable_report must have an action for each report type -_VALID_REPORTS = [{"report" : "host_files", "type" : bool, - "help" : - "Return a list of host CAP files created by capgen"}, - {"report" : "suite_files", "type" : bool, - "help" : - "Return a list of suite CAP files created by capgen"}, - {"report" : "utility_files", "type" : bool, - "help" : ("Return a list of utility files created by " - "capgen (e.g., ccpp_kinds.F90)")}, - {"report" : "ccpp_files", "type" : bool, - "help" : "Return a list of all files created by capgen"}, - {"report" : "process_list", "type" : bool, - "help" : ("Return a list of process types and implementing " - "scheme name")}, - {"report" : "module_list", "type" : bool, - "help" : - "Return a list of module names used in this set of suites"}, - {"report" : "dependencies", "type" : bool, - "help" : ("Return a list of scheme and host " - "dependency module names")}, - {"report" : "suite_list", "type" : bool, - "help" : "Return a list of configured suite names"}, - {"report" : "required_variables", "type" : str, - "help" : ("Return a list of required variable " - "standard names for suite, "), - "metavar" : "SUITE_NAME"}, - {"report" : "input_variables", "type" : str, - "help" : ("Return a list of required input variable " - "standard names for suite, "), - "metavar" : "SUITE_NAME"}, - {"report" : "output_variables", "type" : str, - "help" : ("Return a list of required output variable " - "standard names for suite, "), - "metavar" : "SUITE_NAME"}, - {"report" : "host_variables", "type" : bool, - "help" : ("Return a list of required host model variable " - "standard names")}, - {"report" : "show", "type" : bool, - "help" : - "Pretty print the database contents to the screen"}] +_VALID_REPORTS = [ + {"report": "host_files", "type": bool, + "help": "Return a list of host CAP Fortran files created by capgen"}, + {"report": "suite_files", "type": bool, + "help": "Return a list of suite CAP Fortran files created by capgen"}, + {"report": "scheme_files", "type": bool, + "help": ("Return a list of scheme Fortran source files actually " + "referenced by some loaded suite (group phases + " + "suite-level / hooks). These are the " + "user-supplied scheme .F90 files, NOT capgen-generated " + "caps")}, + {"report": "utility_files", "type": bool, + "help": ("Return a list of utility Fortran files created by " + "capgen (e.g., ccpp_kinds.F90)")}, + {"report": "capgen_files", "type": bool, + "help": ("Return a list of all Fortran files created by capgen " + "(union of --host-files, --suite-files, --utility-files); " + "non-Fortran inspection artifacts are reported via " + "--inspection-files")}, + {"report": "inspection_files", "type": bool, + "help": ("Return a list of non-Fortran inspection files created by " + "capgen (suite .meta files and expanded suite definition " + "XML files)")}, + {"report": "process_list", "type": bool, + "help": ("Return a list of process types and implementing " + "scheme name")}, + {"report": "module_list", "type": bool, + "help": + "Return a list of module names used in this set of suites"}, + {"report": "dependencies", "type": bool, + "help": ("Return a list of scheme and host " + "dependency file paths (from the 'dependencies' " + "attribute in metadata tables)")}, + {"report": "suite_list", "type": bool, + "help": "Return a list of configured suite names"}, + {"report": "required_variables", "type": str, + "help": ("Return a list of required variable " + "standard names for suite, "), + "metavar": "SUITE_NAME"}, + {"report": "input_variables", "type": str, + "help": ("Return a list of required input variable " + "standard names for suite, "), + "metavar": "SUITE_NAME"}, + {"report": "output_variables", "type": str, + "help": ("Return a list of required output variable " + "standard names for suite, "), + "metavar": "SUITE_NAME"}, + {"report": "host_variables", "type": bool, + "help": ("Return a list of required host model variable " + "standard names")}, + {"report": "show", "type": bool, + "help": + "Pretty print the database contents to the screen"}, +] ### ### Utilities ### + class CCPPDatatableError(ValueError): """Error specific to errors found in the CCPP capgen datafile""" pass -class DatatableInternalError(ValueError): - """Error class for reporting internal errors""" - def __init__(self, message): - """Initialize this exception""" - logging.shutdown() - super(DatatableInternalError, self).__init__(message) class DatatableReport(object): """A class to hold a database report type and inquiry function""" @@ -100,7 +114,7 @@ class DatatableReport(object): __valid_actions = [x["report"] for x in _VALID_REPORTS] def __init__(self, action, value=True): - """Initialize this report as report-type, + """Initialize this report as report-type, . # Test a valid action >>> DatatableReport('input_variables', False).action 'input_variables' @@ -117,7 +131,6 @@ def __init__(self, action, value=True): self.__value = value else: raise ValueError("Invalid action, '{}'".format(action)) - # end if def action_is(self, action): """If matches this report type, return True. @@ -145,14 +158,14 @@ def valid_actions(cls): """Return the list of valid actions for this class""" return cls.__valid_actions + ### ### Interface for retrieving datatable information ### -############################################################################### + def _command_line_parser(): -############################################################################### - "Create and return an ArgumentParser for parsing the command line" + """Create and return an ArgumentParser for parsing the command line.""" description = """ Retrieve information about a ccpp_capgen run. The returned information is controlled by selecting an action from @@ -162,7 +175,7 @@ def _command_line_parser(): parser = argparse.ArgumentParser(description=description) parser.add_argument("datatable", type=str, help="Path to a data table XML file created by capgen") - ### Only one action per call + # Only one action per call group = parser.add_mutually_exclusive_group(required=True) for report in _VALID_REPORTS: rep_type = "--{}".format(report["report"].replace("_", "-")) @@ -177,12 +190,8 @@ def _command_line_parser(): else: group.add_argument(rep_type, required=False, type=str, default='', help=report["help"]) - # end if else: raise ValueError("Unknown report type, '{}'".format(report["type"])) - # end if - # end for - ### defval = "," help_str = "String to separate items in a list (default: '{}')" parser.add_argument("--separator", type=str, required=False, default=defval, @@ -206,93 +215,185 @@ def _command_line_parser(): help=help_str.format(defval)) return parser -############################################################################### + def parse_command_line(args): -############################################################################### - """Create an ArgumentParser to parse and return command-line arguments""" + """Create an ArgumentParser to parse and return command-line arguments.""" parser = _command_line_parser() pargs = parser.parse_args(args) return pargs + ### ### Accessor functions to retrieve information from a datatable file ### -############################################################################### + def _read_datatable(datatable): -############################################################################### - """Read the XML file, and return its root node""" - _, datatable = read_xml_file(datatable, None) # No logger - return datatable + """Read XML file *datatable* and return its root node.""" + tree = ET.parse(datatable) + return tree.getroot() + -############################################################################### def _find_table_section(table, elem_type): -############################################################################### """Look for and return an element type, , in . Raise an exception if the element is not found. # Test present section - >>> table = ET.fromstring("") - >>> _find_table_section(table, "ccpp_files").tag - 'ccpp_files' + >>> table = ET.fromstring("") + >>> _find_table_section(table, "capgen_files").tag + 'capgen_files' # Test missing section >>> table = ET.fromstring("") - >>> _find_table_section(table, "ccpp_files").tag + >>> _find_table_section(table, "capgen_files").tag Traceback (most recent call last): ... - ccpp_datafile.CCPPDatatableError: Element type, 'ccpp_files', not found in table + ccpp_datafile.CCPPDatatableError: Element type, 'capgen_files', not found in table """ found = table.find(elem_type) if found is None: emsg = "Element type, '{}', not found in table" raise CCPPDatatableError(emsg.format(elem_type)) - # end if return found -############################################################################### -def _retrieve_ccpp_files(table, file_type=None): -############################################################################### + +def _retrieve_capgen_files(table, file_type=None): """Find and retrieve a list of generated filenames from
. If is not None, only return that file type. # Test valid ccpp files - >>> table = ET.fromstring(""\ + >>> table = ET.fromstring(""\ "/path/to/file1"\ "/path/to/file2"\ "/path/to/file3"\ - "/path/to/file4"\ - "") - >>> _retrieve_ccpp_files(table) + "/path/to/file4"\ + "") + >>> _retrieve_capgen_files(table) ['/path/to/file1', '/path/to/file2', '/path/to/file3', '/path/to/file4'] # Test invalid file type - >>> table = ET.fromstring(""\ + >>> table = ET.fromstring(""\ "/path/to/file1"\ - "") - >>> _retrieve_ccpp_files(table) + "") + >>> _retrieve_capgen_files(table) Traceback (most recent call last): ... ccpp_datafile.CCPPDatatableError: Invalid file list entry type, 'banana' """ - ccpp_files = list() - # Find the files section - for section in _find_table_section(table, "ccpp_files"): + capgen_files = list() + for section in _find_table_section(table, "capgen_files"): if (not file_type) or (section.tag == file_type): for entry in section: if entry.tag == "file": - ccpp_files.append(entry.text) + capgen_files.append(entry.text) else: emsg = "Invalid file list entry type, '{}'" raise CCPPDatatableError(emsg.format(entry.tag)) - # end if - # end for - # end if - # end if - return ccpp_files + return capgen_files + + +def _retrieve_scheme_files(table): + """Find and return the list of used-scheme Fortran source paths from
. + + The ```` section lists the user-supplied scheme ``.F90`` + (or ``.F`` / ``.f90`` / ``.f``) sources for schemes that the loaded + suites actually reference. Build systems use this to compile exactly + the scheme set the suites consume; unreferenced scheme metadata + files passed on the capgen CLI for convenience are filtered out. + + # Test valid scheme files + >>> table = ET.fromstring(""\ + "/path/to/scheme1.F90"\ + "/path/to/scheme2.F90"\ + "") + >>> _retrieve_scheme_files(table) + ['/path/to/scheme1.F90', '/path/to/scheme2.F90'] + + # Test empty + >>> table = ET.fromstring(""\ + "") + >>> _retrieve_scheme_files(table) + [] + + # Test missing section + >>> table = ET.fromstring("") + >>> _retrieve_scheme_files(table) + Traceback (most recent call last): + ... + ccpp_datafile.CCPPDatatableError: Element type, 'scheme_files', not found in table + + # Test invalid entry type + >>> table = ET.fromstring(""\ + "/path/to/scheme1.F90"\ + "") + >>> _retrieve_scheme_files(table) + Traceback (most recent call last): + ... + ccpp_datafile.CCPPDatatableError: Invalid scheme file entry type, 'banana' + """ + result = [] + section = _find_table_section(table, "scheme_files") + for entry in section: + if entry.tag == "file": + if entry.text is not None: + result.append(entry.text) + else: + raise CCPPDatatableError( + "Invalid scheme file entry type, '{}'".format(entry.tag) + ) + return result + + +def _retrieve_inspection_files(table, file_type=None): + """Find and retrieve a list of inspection filenames from
. + + Inspection files are non-Fortran artifacts emitted by capgen for + debugging and downstream tooling: suite ``.meta`` files and expanded + suite-definition XML. Each kind lives in its own subsection of + ````. + + If is not None, only return files in that subsection. + + # Test valid inspection files + >>> table = ET.fromstring(""\ + "/path/to/a.meta"\ + ""\ + "/path/to/a_exp.xml"\ + ""\ + "") + >>> _retrieve_inspection_files(table) + ['/path/to/a.meta', '/path/to/a_exp.xml'] + + # Test file_type filter + >>> _retrieve_inspection_files(table, file_type='suite_meta_files') + ['/path/to/a.meta'] + + # Test invalid entry type + >>> table = ET.fromstring(""\ + "/path/to/a.meta"\ + "") + >>> _retrieve_inspection_files(table) + Traceback (most recent call last): + ... + ccpp_datafile.CCPPDatatableError: Invalid file list entry type, 'banana' + """ + inspection_files = list() + for section in _find_table_section(table, "inspection_files"): + if (not file_type) or (section.tag == file_type): + for entry in section: + if entry.tag == "file": + inspection_files.append(entry.text) + else: + emsg = "Invalid file list entry type, '{}'" + raise CCPPDatatableError(emsg.format(entry.tag)) + return inspection_files + -############################################################################### def _retrieve_process_list(table): -############################################################################### """Find and return a list of all physics scheme processes in
. + + capgen does not currently record a ``process`` attribute on + scheme entries, so this returns an empty list when no scheme carries + one. The flag is kept for CLI compatibility. + # Test valid module >>> table = ET.fromstring(""\ ""\ @@ -308,24 +409,19 @@ def _retrieve_process_list(table): ... ccpp_datafile.CCPPDatatableError: Could not find 'schemes' element """ - result = list() schemes = table.find("schemes") if schemes is None: raise CCPPDatatableError("Could not find 'schemes' element") - # end if for scheme in schemes: name = scheme.get("name") proc = scheme.get("process") if proc: result.append("{}={}".format(proc, name)) - # end if - # end for return sorted(result) -############################################################################### + def _retrieve_module_list(table): -############################################################################### """Find and return a list of all scheme modules in
. # Test valid module >>> table = ET.fromstring(""\ @@ -347,21 +443,18 @@ def _retrieve_module_list(table): schemes = table.find("schemes") if schemes is None: raise CCPPDatatableError("Could not find 'schemes' element") - # end if for scheme in schemes: for phase in scheme: module = phase.get("module") if module is not None: result.add(module) - # end if - # end for - # end for return sorted(result) -############################################################################### + def _retrieve_dependencies(table): -############################################################################### - """Find and return a list of all host and scheme dependencies. + """Find and return a sorted, dedup'd list of host and scheme + dependency file paths (collected from the ``dependencies`` attribute + in metadata tables). # Test valid dependencies >>> table = ET.fromstring("" \ "bananaorange" \ @@ -382,25 +475,20 @@ def _retrieve_dependencies(table): ... ccpp_datafile.CCPPDatatableError: Could not find 'dependencies' element """ - result = set() depends = table.find("dependencies") if depends is None: raise CCPPDatatableError("Could not find 'dependencies' element") - # end if for dependency in depends: dep_file = dependency.text if dep_file is not None: result.add(dep_file) - # end if - # end for return sorted(result) -############################################################################### + def _find_var_dictionary(table, dict_name=None, dict_type=None): -############################################################################### - """Find and return a var_dictionary named, in
. - If not found, return None + """Find and return a var_dictionary in
. + If not found, return None. # Test valid table with dict_name provided >>> table = ET.fromstring(""\ ""\ @@ -433,19 +521,17 @@ def _find_var_dictionary(table, dict_name=None, dict_type=None): if (dict_name is None) and (dict_type is None): raise ValueError(("At least one of or must " "contain a string")) - # end if + if var_dicts is None: + return None for vdict in var_dicts: if (((dict_name is None) or (vdict.get("name") == dict_name)) and ((dict_type is None) or (vdict.get("type") == dict_type))): target_dict = vdict break - # end if - # end for return target_dict -############################################################################### + def _retrieve_suite_list(table): -############################################################################### """Find and return a list of all suites found in
. # Test suites are found >>> table = ET.fromstring(""\ @@ -460,21 +546,16 @@ def _retrieve_suite_list(table): [] """ result = list() - # First, find the API variable dictionary api_elem = table.find("api") if api_elem is not None: suites_elem = api_elem.find("suites") if suites_elem is not None: for suite in suites_elem: result.append(suite.get("name")) - # end for - # end if - # end if return result -############################################################################### + def _retrieve_suite_group_names(table, suite_name): -############################################################################### """Find and return a list of the group names for this suite. # Test suites are found >>> table = ET.fromstring(""\ @@ -489,9 +570,7 @@ def _retrieve_suite_group_names(table, suite_name): >>> _retrieve_suite_group_names(table, 'poncho') [] """ - result = list() - # First, find the API variable dictionary api_elem = table.find("api") if api_elem is not None: suites_elem = api_elem.find("suites") @@ -501,19 +580,12 @@ def _retrieve_suite_group_names(table, suite_name): for item in suite: if item.tag == "group": result.append(item.get("name")) - # end if - # end for - # end if - # end for - # end if - # end if return result -############################################################################### + def _is_variable_protected(table, var_name, var_dict): -############################################################################### """Determine whether variable, , from is protected. - So this by checking for the 'protected' attribute for in + Do this by checking the 'protected' attribute for in or any of 's ancestors (parent dictionaries). # Test found variable >>> table = ET.fromstring(""\ @@ -541,27 +613,21 @@ def _is_variable_protected(table, var_name, var_dict): if var.get("name") == var_name: protected = var.get("protected", default="False") == "True" break - # end if - # end for - # end if parent = var_dict.get("parent") if parent is not None: var_dict = _find_var_dictionary(table, dict_name=parent) else: var_dict = None - # end if - # end while return protected -############################################################################### + def _retrieve_variable_list(table, suite_name, intent_type=None, exclude_protected=True): -############################################################################### """Find and return a list of all the required variables in . If suite, , is not found in
, return an empty list. If is present, return only that variable type (input or output). - If is True, do not include protected variables + If is True, do not include protected variables. >>> table = ET.fromstring(""\ ""\ ""\ @@ -603,8 +669,6 @@ def _retrieve_variable_list(table, suite_name, ccpp_datafile.CCPPDatatableError: Invalid intent_type, 'banana' """ - # Note that suites do not have call lists so we have to collect - # all the variables from the suite's groups. var_set = set() excl_vars = list() if intent_type == "host": @@ -618,7 +682,6 @@ def _retrieve_variable_list(table, suite_name, else: emsg = "Invalid intent_type, '{}'" raise CCPPDatatableError(emsg.format(intent_type)) - # end if if exclude_protected or (intent_type == "host"): host_dict = _find_var_dictionary(table, dict_type="host") if host_dict is not None: @@ -631,22 +694,12 @@ def _retrieve_variable_list(table, suite_name, host_dict) else: exclude = False - # end if if intent_type == "host": if not exclude: - # Add to host variable set var_set.add(vname) - # end if else: if exclude: - # Add to list of protected variables excl_vars.append(vname) - # end if - # end if - # end for - # end if - # end if - # end if if intent_type != "host": group_names = _retrieve_suite_group_names(table, suite_name) for group in group_names: @@ -664,44 +717,36 @@ def _retrieve_variable_list(table, suite_name, if not exclude: exclude = _is_variable_protected(table, vname, group_dict) - # end if else: exclude = False - # end if if (vintent in allowed_intents) and (not exclude): var_set.add(vname) - # end if - # end for - # end if - # end if - # end for - # end if return sorted(var_set) -############################################################################### + def datatable_report(datatable, action, sep, exclude_protected=False): -############################################################################### - """Perform a lookup on and return the result. - """ + """Perform a lookup on and return the result.""" if not action: emsg = "datatable_report: An action is required\n" emsg += _command_line_parser().format_usage() raise ValueError(emsg) - # end if if not sep: emsg = "datatable_report: A separator character () is required\n" emsg += _command_line_parser().format_usage() raise ValueError(emsg) - # end if table = _read_datatable(datatable) - if action.action_is("ccpp_files"): - result = _retrieve_ccpp_files(table) + if action.action_is("capgen_files"): + result = _retrieve_capgen_files(table) elif action.action_is("host_files"): - result = _retrieve_ccpp_files(table, file_type="host_files") + result = _retrieve_capgen_files(table, file_type="host_files") elif action.action_is("suite_files"): - result = _retrieve_ccpp_files(table, file_type="suite_files") + result = _retrieve_capgen_files(table, file_type="suite_files") + elif action.action_is("scheme_files"): + result = _retrieve_scheme_files(table) elif action.action_is("utility_files"): - result = _retrieve_ccpp_files(table, file_type="utilities") + result = _retrieve_capgen_files(table, file_type="utilities") + elif action.action_is("inspection_files"): + result = _retrieve_inspection_files(table) elif action.action_is("process_list"): result = _retrieve_process_list(table) elif action.action_is("module_list"): @@ -722,25 +767,22 @@ def datatable_report(datatable, action, sep, exclude_protected=False): intent_type="output", exclude_protected=exclude_protected) elif action.action_is("host_variables"): - result = _retrieve_variable_list(table, "host", exclude_protected=exclude_protected, + result = _retrieve_variable_list(table, "host", + exclude_protected=exclude_protected, intent_type="host") else: result = '' - # end if if isinstance(result, list): result = sep.join(result) - # end if return result -############################################################################### + def _indent_str(indent): -############################################################################### """Return the line start string for indent level, .""" - return _INDENT_STR*indent + return _INDENT_STR * indent + -############################################################################### def _format_line(line_in, indent, line_wrap, increase_indent=True): -############################################################################### """Format into separate lines in an attempt to not have the length of any line greater than characters including any indent (with indent level specified by ). @@ -769,37 +811,28 @@ def _format_line(line_in, indent, line_wrap, increase_indent=True): wrap_points = list() line = line_in.strip() llen = len(line) - # Do we need to wrap the line? if (line_wrap <= 0) or (llen + curr_indent <= line_wrap): index = llen + 1 else: index = 0 - # end if - # Collect possible wrap points while index < llen: inchar = line[index] if in_squote: if inchar == "'": in_squote = False - # end if (else do nothing) elif in_dquote: if inchar == '"': in_dquote = False - # end if (else do nothing) elif inchar == ' ': wrap_points.append(index + curr_indent) - # end if (else it is not an interesting character) index += 1 - # end while if (line_wrap <= 0) or (llen + curr_indent <= line_wrap): this_line = indent_str + line next_line = "" else: - # Find the best break point good_points = [x for x in wrap_points if x <= line_wrap] if increase_indent: - indent += 2 # To indent past child tags - # end if + indent += 2 if good_points: wrap = max(good_points) - curr_indent this_line = indent_str + line[0:wrap] @@ -813,15 +846,12 @@ def _format_line(line_in, indent, line_wrap, increase_indent=True): else: this_line = indent_str + line next_line = "" - # end if - # end if outline = this_line + '\n' + next_line return outline -############################################################################### + def table_entry_pretty_print(entry, indent, line_wrap=-1): -############################################################################### - """Create and return a pretty print string of the contents of + """Create and return a pretty print string of the contents of . >>> table = ET.fromstring("") >>> table_entry_pretty_print(table, 0) '\\n \\n\\n' @@ -833,382 +863,68 @@ def table_entry_pretty_print(entry, indent, line_wrap=-1): outline = "<{}".format(entry.tag) for name in entry.attrib: outline += " {}={}".format(name, entry.attrib[name]) - # end for has_children = len(list(entry)) > 0 has_text = entry.text if has_children or has_text: - # We have sub-structure, close and print this tag outline += ">" output += _format_line(outline, indent, line_wrap) else: - # No sub-structure, we are done with this tag outline += " />" output += _format_line(outline, indent, line_wrap) - # end if if has_children: for child in entry: output += table_entry_pretty_print(child, indent+1, line_wrap=line_wrap) - # end for - # end if if has_text: output += _format_line(entry.text, indent+1, line_wrap) - # end if if has_children or has_text: - # We had sub-structure, print the close tag outline = "".format(entry.tag) output = output.rstrip() + '\n' + _format_line(outline, indent, line_wrap) - # end if return output -############################################################################### + def datatable_pretty_print(datatable, indent, line_wrap): -############################################################################### - """Create and return a pretty print string of the contents of """ + """Create and return a pretty print string of the contents of .""" indent = 0 table = _read_datatable(datatable) report = table_entry_pretty_print(table, indent, line_wrap=line_wrap) return report + ### -### Functions to create the datatable file +### Main entry point ### -############################################################################### -def _object_type(pobj): -############################################################################### - """Return an XML-acceptable string for the type of .""" - return pobj.__class__.__name__.lower() - -############################################################################### -def _new_var_entry(parent, var, full_entry=True): -############################################################################### - """Create a variable sub-element of with information from . - If is False, only include standard name and intent. - >>> parent = ET.fromstring('') - >>> var = Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '(horizontal_loop_extent)', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'DDT', ParseContext()), _MVAR_DUMMY_RUN_ENV) - >>> _new_var_entry(parent, var) - >>> table_entry_pretty_print(parent, 0) - '\\n \\n \\n horizontal_loop_extent\\n \\n \\n ddt\\n \\n \\n vname\\n \\n \\n\\n' - - >>> parent = ET.fromstring('') - >>> _new_var_entry(parent, var, full_entry=False) - >>> table_entry_pretty_print(parent, 0) - '\\n \\n\\n' - """ - prop_list = ["intent", "local_name"] - if full_entry: - prop_list.extend(["allocatable", "active", "default_value", - "diagnostic_name", "diagnostic_name_fixed", - "kind", "persistence", "polymorphic", "protected", - "state_variable", "type", "units", "molar_mass", - "advected", "top_at_one", "optional"]) - prop_list.extend(Var.constituent_property_names()) - # end if - ventry = ET.SubElement(parent, "var") - ventry.set("name", var.get_prop_value("standard_name")) - for prop in prop_list: - value = var.get_prop_value(prop) - if value: - ventry.set(prop, str(value)) - # end if - # end for - if full_entry: - dims = var.get_dimensions() - if dims: - v_entry = ET.SubElement(ventry, "dimensions") - v_entry.text = " ".join(dims) - # end if - v_entry = ET.SubElement(ventry, "source_type") - v_entry.text = var.source.ptype.lower() - v_entry = ET.SubElement(ventry, "source_name") - v_entry.text = var.source.name.lower() - # end if - -############################################################################### -def _new_scheme_entry(parent, scheme, group_name, scheme_headers): -############################################################################### - """Create a new XML entry for under """ - sch_name = scheme.name - sch_entry = parent.find(sch_name) - process = None - if not sch_entry: - sch_entry = ET.SubElement(parent, "scheme") - sch_entry.set("name", sch_name) - # end if - if scheme.run_phase(): - sch_tag = group_name - else: - sch_tag = scheme.phase() - # end if - if not sch_tag: - emsg = "No phase info for scheme, '{}', group = '{}" - raise CCPPDatatableError(emsg.format(sch_name, group_name)) - # end if - phase_entry = sch_entry.find(sch_tag) - if phase_entry: - pname = phase_entry.get("name") - if pname != sch_name: - emsg = "Scheme entry already exists for {} but name is {}" - raise CCPPDatatableError(emsg.format(sch_name, pname)) - # end if - # Special case: Scheme w/o run phase. - if not scheme._has_run_phase: - return + +def main(argv: Optional[List[str]] = None) -> int: + global _INDENT_STR + if argv is None: + argv = sys.argv[1:] + pargs = parse_command_line(argv) + if pargs.show: + _INDENT_STR = " " * pargs.indent + report = datatable_pretty_print(pargs.datatable, 0, + line_wrap=pargs.line_wrap) else: - phase_entry = ET.SubElement(sch_entry, sch_tag) - phase_entry.set("name", sch_name) - title = scheme.subroutine_name - phase_entry.set("subroutine_name", title) - phase_entry.set("filename", scheme.context.filename) - if title in scheme_headers: - header = scheme_headers[title] - proc = header.process_type - if proc != UNKNOWN_PROCESS_TYPE: - if process: - if process != proc: - emsg = 'Inconsistent process, {} != {}' - raise CCPPDatatableError(emsg.format(proc, process)) - # end if (no else, things are okay) + arg_vars = vars(pargs) + action = None + errmsg = '' + esep = '' + for opt in arg_vars: + if (opt in DatatableReport.valid_actions()) and arg_vars[opt]: + if action: + errmsg += esep + "Duplicate action, '{}'".format(opt) + esep = '\n' else: - process = proc - # end if - # end if - module = header.module - if module: - phase_entry.set("module", module) - # end if - else: - emsg = 'Could not find metadata header for {}' - raise CCPPDatatableError(emsg.format(sch_name)) - # end if - call_list = ET.SubElement(phase_entry, "call_list") - vlist = scheme.call_list.variable_list() - for var in vlist: - _new_var_entry(call_list, var, full_entry=False) - # end for - # end if - if process: - sch_entry.set("process", proc) - # end if - -############################################################################### -def _new_variable_dictionary(dictionaries, var_dict, dict_type, parent=None): -############################################################################### - """Create a new XML entry for under .""" - dict_entry = ET.SubElement(dictionaries, "var_dictionary") - dict_entry.set("name", var_dict.name) - dict_entry.set("type", dict_type) - if parent is not None: - dict_entry.set("parent", parent.name) - # end if - sub_dicts = var_dict.sub_dictionaries() - if sub_dicts: - sd_entry = ET.SubElement(dict_entry, "sub_dictionaries") - sd_entry.text = " ".join([x.name for x in sub_dicts]) - # end if - vars_entry = ET.SubElement(dict_entry, "variables") - for var in var_dict.variable_list(): - _new_var_entry(vars_entry, var, full_entry=True) - # end for - -############################################################################### -def _add_suite_object_dictionaries(dictionaries, suite_object): -############################################################################### - """Create new XML entries for under . - Add 's dictionary and its call_list dictionary (if present). - Recurse to this objects parts.""" - dict_type = _object_type(suite_object) - _new_variable_dictionary(dictionaries, suite_object, dict_type, - parent=suite_object.parent) - if suite_object.call_list: - dict_type += "_call_list" - _new_variable_dictionary(dictionaries, suite_object.call_list, - dict_type, parent=suite_object.parent) - # end if - for part in suite_object.parts: - _add_suite_object_dictionaries(dictionaries, part) - # end for - -############################################################################### -def _add_dependencies(parent, scheme_depends, host_depends): -############################################################################### - """Add a section to that lists all the dependencies - required by schemes or the host model. - >>> parent = ET.fromstring("") - >>> scheme_depends = ['file1', 'file2'] - >>> host_depends = ['file3', 'file4'] - >>> _add_dependencies(parent, scheme_depends, host_depends) - >>> table_entry_pretty_print(parent, 0) - '\\n \\n \\n file3\\n \\n \\n file4\\n \\n \\n file1\\n \\n \\n file2\\n \\n \\n\\n' - """ - file_entry = ET.SubElement(parent, "dependencies") - for hfile in host_depends: - entry = ET.SubElement(file_entry, "dependency") - entry.text = hfile - # end for - for sfile in scheme_depends: - entry = ET.SubElement(file_entry, "dependency") - entry.text = sfile - # end for - -############################################################################### -def _add_generated_files(parent, host_files, suite_files, ccpp_kinds, src_dir): -############################################################################### - """Add a section to that lists all the files generated - by in sections for host cap, suite caps, ccpp_kinds, and source files. - Also add existing utility files which are always needed by the framework. - """ - file_entry = ET.SubElement(parent, "ccpp_files") - utilities = ET.SubElement(file_entry, "utilities") - entry = ET.SubElement(utilities, "file") - entry.text = ccpp_kinds - entry = ET.SubElement(utilities, "file") - entry.text = os.path.join(src_dir, "ccpp_constituent_prop_mod.F90") - entry = ET.SubElement(utilities, "file") - entry.text = os.path.join(src_dir, "ccpp_scheme_utils.F90") - entry = ET.SubElement(utilities, "file") - entry.text = os.path.join(src_dir, "ccpp_hashable.F90") - entry = ET.SubElement(utilities, "file") - entry.text = os.path.join(src_dir, "ccpp_hash_table.F90") - host_elem = ET.SubElement(file_entry, "host_files") - for hfile in host_files: - entry = ET.SubElement(host_elem, "file") - entry.text = hfile - # end for - suite_elem = ET.SubElement(file_entry, "suite_files") - for sfile in suite_files: - entry = ET.SubElement(suite_elem, "file") - entry.text = sfile - # end for - -############################################################################### -def _add_suite_object(parent, suite_object): -############################################################################### - """Add an entry for under . This operation is - recursive to all the components inside of """ - obj_elem = ET.SubElement(parent, _object_type(suite_object)) - obj_elem.set("name", suite_object.name) - ptype = suite_object.phase_type - if ptype: - obj_elem.set("phase", ptype) - # end if - if isinstance(suite_object, Subcycle): - obj_elem.set("loop", suite_object._loop) - # end if - for obj_part in suite_object.parts: - _add_suite_object(obj_elem, obj_part) - # end for - -############################################################################### -def generate_ccpp_datatable(run_env, host_model, api, scheme_headers, - scheme_tdict, host_files, suite_files, - ccpp_kinds, source_dir): -############################################################################### - """Write a CCPP datatable for to . - The datatable includes the generated filenames for the host cap, - the suite caps, the ccpp_kinds module, and source code files. - """ - # Define new tree - datatable = ET.Element("ccpp_datatable") - datatable.set("version", "1.0") - # Write out the generated files - _add_generated_files(datatable, host_files, suite_files, - ccpp_kinds, source_dir) - # Write out scheme info - schemes = ET.SubElement(datatable, "schemes") - # Create a dictionary of the scheme headers for easy lookup - scheme_header_dict = {} - for header in scheme_headers: - if header.title in scheme_header_dict: - emsg = 'Header {} already in dictionary' - raise CCPPDatatableError(emsg.format(header.title)) - # end if - scheme_header_dict[header.title] = header - # end for - # Dump all scheme info from the suites - for suite in api.suites: - for group in suite.groups: - gname = group.name - for scheme in group.schemes(): - _new_scheme_entry(schemes, scheme, gname, scheme_header_dict) - # end for - # end for - # end for - # Write the API - api_elem = ET.SubElement(datatable, "api") - suites_elem = ET.SubElement(api_elem, "suites") - for suite in api.suites: - suite_elem = ET.SubElement(suites_elem, "suite") - suite_elem.set("name", suite.name) - suite_elem.set("filename", suite.sdf_name) - for group in suite.groups: - # Skip empty groups - if group.parts: - _add_suite_object(suite_elem, group) - # end if - # end for - # end for - # Dump the variable dictionaries - var_dicts = ET.SubElement(datatable, "var_dictionaries") - # First, the top-level dictionaries - _new_variable_dictionary(var_dicts, host_model, "host") - _new_variable_dictionary(var_dicts, api, "api", parent=api.parent) - # Now, the suite and group namelists, etc. (including call_lists) - for suite in api.suites: - _new_variable_dictionary(var_dicts, suite, "suite", parent=suite.parent) - for group in suite.groups: - _add_suite_object_dictionaries(var_dicts, group) - # end for - # end for - # end for - # Add in all dependencies - scheme_depends = set() - for table in scheme_tdict: - for dep_file in scheme_tdict[table].dependencies: - scheme_depends.add(dep_file) - # end for - # end for - host_depends = set() - host_tables = host_model.metadata_tables() - for table in host_tables: - for dep_file in host_tables[table].dependencies: - host_depends.add(dep_file) - # end for - # end for - _add_dependencies(datatable, scheme_depends, host_depends) - # Write tree - write_xml_file(datatable, run_env.datatable_file) - -############################################################################### + action = DatatableReport(opt, arg_vars[opt]) + if errmsg: + raise ValueError(errmsg) + report = datatable_report(pargs.datatable, action, + pargs.sep, pargs.exclude_protected) + print("{}".format(report.rstrip())) + return 0 + if __name__ == "__main__": - PARGS = parse_command_line(sys.argv[1:]) - if PARGS.show: - _INDENT_STR = " "*PARGS.indent - LINE_WRAP = PARGS.line_wrap - REPORT = datatable_pretty_print(PARGS.datatable, 0, line_wrap=LINE_WRAP) - else: - ARG_VARS = vars(PARGS) - _ACTION = None - _ERRMSG = '' - _ESEP = '' - for opt in ARG_VARS: - if (opt in DatatableReport.valid_actions()) and ARG_VARS[opt]: - if _ACTION: - _ERRMSG += _ESEP + "Duplicate action, '{}'".format(opt) - _ESEP = '\n' - else: - _ACTION = DatatableReport(opt, ARG_VARS[opt]) - # end if - # end if - # end for - if _ERRMSG: - raise ValueError(_ERRMSG) - # end if - REPORT = datatable_report(PARGS.datatable, _ACTION, - PARGS.sep, PARGS.exclude_protected) - # end if - print("{}".format(REPORT.rstrip())) - sys.exit(0) + sys.exit(main()) diff --git a/capgen/ccpp_validator.py b/capgen/ccpp_validator.py new file mode 100755 index 00000000..1247e250 --- /dev/null +++ b/capgen/ccpp_validator.py @@ -0,0 +1,1832 @@ +#!/usr/bin/env python3 + +"""ccpp_validator — validate Fortran source files against CCPP scheme metadata. + +For each scheme phase declared in a ``.meta`` file this tool checks that the +corresponding Fortran subroutine: + +1. **Exists** in the Fortran source tree. +2. Has the **same number of dummy arguments** as declared in the metadata. +3. The dummy-argument **names match** the ``local_name`` values in the metadata + (order-insensitive). +4. For every dummy argument present in both sides, the **per-arg attributes** + agree: ``intent``, ``type``, ``kind``, and number of dimensions (rank). + ``character`` length must be declared CONSISTENTLY — the metadata mirrors + the Fortran exactly, so ``len=*`` matches only ``len=*`` and ``len=N`` only + the identical ``len=N`` (no wildcarding). Old-style F77 forms + (``character*64``, ``character*(*)``, ``c*5``) are normalised to the + ``len=`` form before comparison. Additionally, host / DDT metadata passed + via ``--host-files`` *defines* its character storage, so ``len=*`` is + rejected there outright (a concrete ``len=N`` required); see + :func:`_check_definition_character_lengths`. Control tables are exempt. + +Asymmetric treatment of ``optional``: + +* Fortran-declared optional argument **absent** from metadata → silently + allowed (the cap never passes it); emits a ``logger.warning``. +* Fortran-declared optional argument **present** in metadata as + ``optional=False`` → silently allowed (the cap always passes it, which + is a valid subset of the Fortran contract); emits a ``logger.warning``. +* Metadata declares ``optional=True`` but Fortran does **not** carry the + ``optional`` attribute → **error** (the cap-side ``present()`` check + would be invalid on a Fortran-required dummy). + +The tool does *not* compare dimension *bounds* across sides — it only +checks that rank matches. Comparing standard-name dimension references +against Fortran local-name dimensions would require loading host metadata +too; that's a separate feature. + +Usage +----- +:: + + ccpp_validator.py \\ + --scheme-files scheme1.meta,scheme2.meta \\ + --source-files scheme1.F90,scheme2.F90 \\ + [--verbose] + +Exit codes +---------- +0 — all checks passed +1 — one or more validation errors found +2 — internal / usage error +""" + +import argparse +import logging +import os +import re +import sys +from typing import Dict, List, NamedTuple, Optional, Set, Tuple + +# Ensure the capgen package is importable when invoked directly. +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +_PACKAGE_DIR = os.path.dirname(_SCRIPT_DIR) +if _PACKAGE_DIR not in sys.path: + sys.path.insert(0, _PACKAGE_DIR) + +from metadata.parse_tools import CCPPError, init_log, set_log_level +from metadata.variable_resolver import SchemeStore +from metadata.metadata_table import parse_metadata_file + +_LOGGER = init_log('ccpp_validator') + +# --------------------------------------------------------------------------- +# Fortran source parsing helpers +# --------------------------------------------------------------------------- + +# Matches the start of a subroutine definition (case-insensitive, optional +# prefixes recursive/pure/elemental). +_SUB_RE = re.compile( + r'(?i)\s*(?:(?:recursive|pure|elemental|impure)\s+)*' + r'subroutine\s+(\w+)\s*(?:\(([^)]*)\))?' +) +# Matches `end subroutine [name]` (case-insensitive). Bare ``end`` is not +# detected — CCPP scheme code consistently uses the explicit form. +_END_SUB_RE = re.compile(r'(?i)^\s*end\s*subroutine\b') +_CONT_RE = re.compile(r'&\s*(?:!.*)?$') # Fortran line continuation +_COMMENT_RE = re.compile(r'!.*$') +# Fixed-form continuation marker. F77 / fixed-form Fortran requires a +# non-blank character in column 6 of a continuation line; CCPP code +# conventionally uses ``&`` for this and pairs it with a trailing ``&`` +# on the prior line for portability with free-form parsers. Only +# applied when we know we are mid-continuation (the buffer is non-empty). +_LEAD_CONT_RE = re.compile(r'^\s*&\s?') +# Matches an identifier-character anywhere in a string. Used by the +# decoration-repair branch to distinguish "stray punctuation past a +# trailing ``&``" (safe to drop) from "real tokens past a ``&``" +# (leave alone so the parser surfaces a real error). +_IDENT_CHAR_RE = re.compile(r'[A-Za-z_0-9]') + + +class _ArgAttrs(NamedTuple): + """Per-dummy-argument attributes parsed from a Fortran type-decl line. + + All string fields are lowercased and stripped. Missing / unknown is + represented by an empty string (or ``False`` for ``optional``, ``0`` + for ``rank``). + + Attributes + ---------- + type_ : str + Intrinsic type (``'real'``, ``'integer'``, ``'logical'``, + ``'complex'``, ``'character'``) or a derived-type spec + (``'type(my_type)'``, ``'class(other)'``). Empty for args + not declared in the body (parser missed the decl). + kind_ : str + Kind selector. For numeric types this is the kind name + (``'kind_phys'``, ``'8'``, ``'int64'``). For character it is + the length selector (``'len=10'``, ``'len=*'``, ``'len=:'``). + Empty when no selector was present. + intent : str + ``'in'`` / ``'out'`` / ``'inout'``, or ``''`` when no intent was + declared (treated as INOUT by Fortran, but for validation we + prefer to flag the absence explicitly). + optional : bool + True iff the type-decl line carried the ``optional`` attribute. + rank : int + Number of dimensions. Computed from ``dimension(...)`` on the + line, or from ``var(:,:,...)``-style trailing parens on the + variable token. ``0`` for scalar. + """ + type_: str + kind_: str + intent: str + optional: bool + rank: int + + +class _SubSig(NamedTuple): + """Parsed signature of one Fortran subroutine. + + Attributes + ---------- + args : list of str + Lowercase dummy-argument names in declaration order. + optional : set of str + Subset of *args* declared with the ``optional`` attribute in the + subroutine body. These args may be absent from the metadata + without producing a validation error — they will simply never be + passed at the cap call site. + attrs : dict + Mapping of lowercase arg name to :class:`_ArgAttrs`. Args + whose type-decl line couldn't be parsed are absent from the + dict; ``_validate_arg_attributes`` skips per-attribute checks + for those args (the name-set check still applies). + """ + args: List[str] + optional: Set[str] + attrs: Dict[str, '_ArgAttrs'] + + +def _paren_aware_split(s: str, sep: str) -> List[str]: + """Split *s* on *sep*, ignoring separators inside balanced parentheses. + + Examples + -------- + >>> _paren_aware_split('integer, optional, intent(in)', ',') + ['integer', ' optional', ' intent(in)'] + >>> _paren_aware_split('x, y(:,:), z', ',') + ['x', ' y(:,:)', ' z'] + """ + result: List[str] = [] + depth = 0 + buf = '' + for ch in s: + if ch == '(': + depth += 1 + buf += ch + elif ch == ')': + depth -= 1 + buf += ch + elif ch == sep and depth == 0: + result.append(buf) + buf = '' + else: + buf += ch + if buf: + result.append(buf) + return result + + +_INTENT_RE = re.compile(r'(?i)^intent\s*\(\s*(in\s*out|inout|in|out)\s*\)\s*$') +_DIM_ATTR_RE = re.compile(r'(?i)^dimension\s*\(\s*(.*?)\s*\)\s*$') +_TYPE_SPEC_RE = re.compile( + r'(?i)^\s*(real|integer|logical|complex|character|double\s*precision' + r'|type\s*\([^)]*\)|class\s*\([^)]*\))\s*(\(.*\))?\s*$' +) +_KIND_SELECTOR_RE = re.compile(r'(?i)^\(\s*(.*?)\s*\)$') + + +def _split_type_spec(spec: str) -> "Tuple[str, str]": + """Split a Fortran type spec into ``(type, kind)``. + + The type is lowercased; the kind selector is left in its raw form + (lowercased, whitespace stripped). Returns ``('', '')`` if *spec* + isn't a recognised type spec. + + Examples + -------- + >>> _split_type_spec('real') + ('real', '') + >>> _split_type_spec('real(kind=kind_phys)') + ('real', 'kind_phys') + >>> _split_type_spec('real(kind_phys)') + ('real', 'kind_phys') + >>> _split_type_spec('real(8)') + ('real', '8') + >>> _split_type_spec('integer(int64)') + ('integer', 'int64') + >>> _split_type_spec('character(len=10)') + ('character', 'len=10') + >>> _split_type_spec('character(len=*)') + ('character', 'len=*') + >>> _split_type_spec('character(*)') + ('character', 'len=*') + >>> _split_type_spec('character') + ('character', '') + >>> _split_type_spec('character*64') + ('character', 'len=64') + >>> _split_type_spec('character*(*)') + ('character', 'len=*') + >>> _split_type_spec('character*(80)') + ('character', 'len=80') + >>> _split_type_spec('type(my_t)') + ('type(my_t)', '') + >>> _split_type_spec('double precision') + ('double precision', '') + >>> _split_type_spec('not_a_type') + ('', '') + """ + spec = spec.strip() + # Old-style (F77) character length given with ``*`` instead of a modern + # ``(len=...)`` selector: + # character*64 -> len=64 character*(*) -> len=* + # character*(80) -> len=80 character*(CL) -> len=cl (named) + m_old = re.match(r'(?i)^character\s*\*\s*(.+)$', spec) + if m_old is not None: + length = m_old.group(1).strip() + paren = re.match(r'^\(\s*(.*?)\s*\)$', length) # peel one ( ) layer + if paren is not None: + length = paren.group(1).strip() + if length == '*': + return ('character', 'len=*') + return ('character', 'len={}'.format(length.lower())) + m = _TYPE_SPEC_RE.match(spec) + if m is None: + return ('', '') + type_raw = m.group(1).lower() + kind_paren = m.group(2) or '' + # Normalise whitespace inside "double precision". + type_ = re.sub(r'\s+', ' ', type_raw) + if type_.startswith('type(') or type_.startswith('class('): + # Strip whitespace inside the parens. + type_ = re.sub(r'\s+', '', type_) + return (type_, '') + if not kind_paren: + return (type_, '') + inner_match = _KIND_SELECTOR_RE.match(kind_paren.strip()) + if inner_match is None: + return (type_, '') + inner = inner_match.group(1).strip() + if type_ == 'character': + # character has its own selector grammar. Accept: + # * -> len=* + # len=... -> len=... + # -> len= + # len=...,kind=... -> use the len= portion + if inner == '*': + return ('character', 'len=*') + if inner.lower().startswith('len='): + # Strip trailing ",kind=..." if present. + len_part = inner.split(',')[0].strip() + return ('character', len_part.lower()) + if re.match(r'^\d+$', inner) or inner == ':': + return ('character', 'len={}'.format(inner)) + # Anything else: store raw, lowercased. + return ('character', inner.lower()) + # Numeric types: accept ``kind=`` or bare ````. + if inner.lower().startswith('kind='): + inner = inner[len('kind='):].strip() + return (type_, inner.lower()) + + +def _parse_decl_line(line: str) -> Dict[str, _ArgAttrs]: + """Parse a Fortran type-declaration line into per-name attributes. + + Returns a (possibly empty) mapping ``{lower_name: _ArgAttrs}``. Lines + that aren't type declarations (no ``::``) or whose type spec doesn't + parse return ``{}``. + + Examples + -------- + >>> attrs = _parse_decl_line('integer, intent(in) :: im') + >>> attrs['im'] + _ArgAttrs(type_='integer', kind_='', intent='in', optional=False, rank=0) + >>> attrs = _parse_decl_line('real(kind=kind_phys), intent(inout) :: temp(:,:)') + >>> attrs['temp'] + _ArgAttrs(type_='real', kind_='kind_phys', intent='inout', optional=False, rank=2) + >>> attrs = _parse_decl_line('character(len=*), intent(out) :: errmsg') + >>> attrs['errmsg'] + _ArgAttrs(type_='character', kind_='len=*', intent='out', optional=False, rank=0) + >>> attrs = _parse_decl_line('real, optional, intent(in), dimension(:) :: a, b(:,:), c') + >>> sorted(attrs.items()) + [('a', _ArgAttrs(type_='real', kind_='', intent='in', optional=True, rank=1)), ('b', _ArgAttrs(type_='real', kind_='', intent='in', optional=True, rank=2)), ('c', _ArgAttrs(type_='real', kind_='', intent='in', optional=True, rank=1))] + >>> _parse_decl_line(' ! comment :: not a decl') + {} + >>> _parse_decl_line('integer :: only_local') + {'only_local': _ArgAttrs(type_='integer', kind_='', intent='', optional=False, rank=0)} + >>> _parse_decl_line('character*256, intent(out) :: scheme_name')['scheme_name'] + _ArgAttrs(type_='character', kind_='len=256', intent='out', optional=False, rank=0) + >>> sorted(_parse_decl_line('character :: c*5, d(10)*8').items()) + [('c', _ArgAttrs(type_='character', kind_='len=5', intent='', optional=False, rank=0)), ('d', _ArgAttrs(type_='character', kind_='len=8', intent='', optional=False, rank=1))] + """ + line = _COMMENT_RE.sub('', line) + if '::' not in line: + return {} + before, _, after = line.partition('::') + tokens = _paren_aware_split(before, ',') + if not tokens: + return {} + type_, kind_ = _split_type_spec(tokens[0]) + if not type_: + return {} + intent = '' + optional = False + line_rank = 0 + for tok in tokens[1:]: + t = tok.strip() + tl = t.lower() + if tl == 'optional': + optional = True + continue + m = _INTENT_RE.match(t) + if m: + iv = m.group(1).lower().replace(' ', '') + intent = iv # 'in' / 'out' / 'inout' + continue + m = _DIM_ATTR_RE.match(t) + if m: + line_rank = len(_paren_aware_split(m.group(1), ',')) + continue + # Anything else (allocatable, pointer, target, parameter, save, + # public, private, contiguous, asynchronous, volatile, value) is + # ignored — we only validate the attrs metadata declares. + result: Dict[str, _ArgAttrs] = {} + for var_tok in _paren_aware_split(after, ','): + var_tok = var_tok.strip() + if not var_tok: + continue + # Strip any ``= `` (or ``=> ``) clause before the + # name regex. A Fortran array initialiser may carry a full + # ``(/ ..., ..., ... /)`` constructor whose inner commas would + # otherwise be gobbled by the regex's greedy parens-matcher and + # miscounted as rank-N entries. ``=`` characters inside other + # parenthesised sub-expressions (e.g. ``::x = (a==b)``) live at + # depth > 0 and are skipped. + var_tok = _strip_initialiser(var_tok).rstrip() + # Old-style (F77) per-entity character length: ``c*5`` / ``d(10)*8`` + # / ``s*(*)``. A trailing ``*`` overrides the type-spec + # length for THIS entity only. + entity_kind = None + m_star = re.match( + r'(?i)^(.*?)\s*\*\s*(\(\s*\*\s*\)|\(\s*\w+\s*\)|\d+|\*|\w+)\s*$', + var_tok, + ) + if m_star is not None: + var_tok = m_star.group(1).strip() + length = m_star.group(2).strip() + paren = re.match(r'^\(\s*(.*?)\s*\)$', length) + if paren is not None: + length = paren.group(1).strip() + entity_kind = 'len=*' if length == '*' else 'len={}'.format(length.lower()) + name_match = re.match(r'(\w+)\s*(\((.*)\))?\s*$', var_tok) + if name_match is None: + continue + name = name_match.group(1).lower() + inner = name_match.group(3) + if inner is not None: + rank = len(_paren_aware_split(inner, ',')) + else: + rank = line_rank + result[name] = _ArgAttrs( + type_=type_, + kind_=entity_kind if entity_kind is not None else kind_, + intent=intent, optional=optional, rank=rank, + ) + return result + + +def _strip_initialiser(var_tok: str) -> str: + """Return *var_tok* with any trailing ``= `` removed. + + Paren-aware: only an ``=`` (or the first ``=`` of ``=>``) at nesting + depth zero is treated as the start of an initialiser. ``=`` + characters inside parenthesised sub-expressions (such as array + initialisers ``(/ ... /)`` or default expressions ``(a == b)``) + are skipped. + + Examples + -------- + >>> _strip_initialiser('foo') + 'foo' + >>> _strip_initialiser('foo(:)') + 'foo(:)' + >>> _strip_initialiser('p => null()') + 'p ' + >>> _strip_initialiser('arr(n) = (/ 1, 2, 3 /)') + 'arr(n) ' + >>> _strip_initialiser('std_name_array(num_consts) = (/ ' + ... "'a', 'b', 'c' /)") + 'std_name_array(num_consts) ' + """ + depth = 0 + for i, ch in enumerate(var_tok): + if ch == '(': + depth += 1 + elif ch == ')': + depth -= 1 + elif ch == '=' and depth == 0: + return var_tok[:i] + return var_tok + + +def _join_continuation( + lines: List[str], + filename: Optional[str] = None, +) -> List[str]: + """Join Fortran continuation lines (ending with ``&``) into single logical lines. + + Handles four continuation conventions seen in real CCPP physics code: + + * **Free-form**: ``&`` only at the trailing end of the prior line. + * **Dual-form**: ``&`` at the trailing end of the prior line *and* + at column 6 of the next line. In this case the leading ``&`` is + part of the continuation marker, not the continued expression, + and is stripped before the next line is appended to the buffer. + * **Fixed-form leading-only**: NO trailing ``&`` on the prior line, + but a ``&`` (or any non-blank) at column 6 of the next line. F77 + / fixed-form Fortran treats this as a continuation; CCPP physics + occasionally relies on it (e.g. ``sfc_sice.f``'s ``sfc_sice_run`` + signature, where the line before the closing ``)`` has no trailing + ``&``). Detected by look-ahead at the next non-blank, non-comment + line. + * **Decorated trailing ``&``** (repair): a ``&`` near the end of the + line is followed by stray non-identifier characters (commas, + parens, whitespace) — typically a typo or hand-edit artefact that + compilers silently ignore because it lives past column 72 in + strict fixed-form mode. When the next line's column-6 ``&`` + already proves we are mid-continuation, treat the last ``&`` as + the continuation marker, discard the decoration, and emit a + ``logger.warning`` naming *filename* so the user knows their + source has decoration past the statement end. + + Parameters + ---------- + lines : list of str + Source lines, each ending in ``\\n`` (as from ``splitlines(keepends=True)``). + filename : str, optional + Source path used only in the decoration-repair warning message. + Defaults to ```` when not supplied. + + Examples + -------- + >>> _join_continuation([' foo &\\n', ' bar\\n', ' baz\\n']) + [' foo bar', ' baz'] + >>> _join_continuation([' foo &\\n', ' & bar\\n', ' baz\\n']) + [' foo bar', ' baz'] + >>> _join_continuation([' foo &\\n', ' & bar\\n', + ... ' & )\\n', ' baz\\n']) + [' foo bar )', ' baz'] + """ + # First pass: normalise each line — strip trailing newlines and any + # inline ``!`` comment. Keep blank/comment-only lines as ``''`` so + # we can skip them when buffering and still use their position for + # look-ahead. + norm: List[str] = [] + for raw in lines: + line = raw.rstrip('\n').rstrip('\r') + norm.append(_COMMENT_RE.sub('', line)) + + def _next_starts_with_lead_cont(start_idx: int) -> bool: + """True iff the next non-blank, non-comment line begins with a + leading ``&`` (fixed-form column-6 continuation marker).""" + j = start_idx + 1 + while j < len(norm) and not norm[j].strip(): + j += 1 + return j < len(norm) and bool(_LEAD_CONT_RE.match(norm[j])) + + result: List[str] = [] + buf = '' + for i, stripped in enumerate(norm): + if buf and not stripped.strip(): + # Mid-continuation, and this line is blank or a pure + # comment. Fortran 90+ permits such lines interleaved + # between continuation lines without ending the logical + # line — skip and keep accumulating. + continue + if buf: + # Mid-continuation: drop a leading ``&`` (fixed-form + # column-6 marker) so it doesn't end up glued into the + # continued expression. No-op on free-form code. + stripped = _LEAD_CONT_RE.sub('', stripped, count=1) + has_trailing = bool(_CONT_RE.search(stripped)) + if has_trailing: + buf += _CONT_RE.sub('', stripped) + continue + # No trailing ``&`` — but a fixed-form continuation may still + # be implied by the next line's column-6 ``&``. If so, keep + # buffering rather than flushing, and try the decoration-repair + # in case the trailing ``&`` was decorated with stray punctuation + # that lives past column 72 (compilers silently drop it; we'd + # otherwise glue it into the joined statement). + if _next_starts_with_lead_cont(i): + stripped = _repair_decorated_trailing_amp(stripped, filename, i + 1) + buf += stripped + continue + buf += stripped + result.append(buf) + buf = '' + if buf: + result.append(buf) + return result + + +def _repair_decorated_trailing_amp( + line: str, + filename: Optional[str], + line_no: int, +) -> str: + """Strip a decorated trailing ``&`` from *line*. + + Called from the fixed-form look-ahead branch of + :func:`_join_continuation`, where the next line's column-6 ``&`` has + already established that we are mid-continuation. If *line* + contains a ``&`` followed only by non-identifier characters + (commas, parens, semicolons, whitespace), the ``&`` is the + decorated continuation marker — drop everything from it onward and + emit a single ``WARNING`` so the user sees that their source has + decoration the compiler is silently ignoring. + + If *line* contains no ``&`` (true fixed-form-leading-only + continuation), or if any token past the last ``&`` looks like a + real Fortran identifier, the line is returned unchanged so the + parser can surface a real error. + """ + amp_idx = line.rfind('&') + if amp_idx < 0: + return line + trailing = line[amp_idx + 1:].strip() + if not trailing: + # ``_CONT_RE`` should have caught this; defensive no-op. + return line + if _IDENT_CHAR_RE.search(trailing): + return line + _LOGGER.warning( + "%s:%d: dropping decoration past trailing '&' (%r); " + "compiler silently ignores this but the parser would otherwise " + "glue it into the statement", + filename or '', line_no, line[amp_idx:], + ) + return line[:amp_idx] + + +def _parse_subroutines( + source: str, + filename: Optional[str] = None, +) -> Dict[str, _SubSig]: + """Extract subroutine signatures from Fortran *source*. + + Returns a mapping ``{subroutine_name_lower: _SubSig}`` where each + :class:`_SubSig` carries the dummy-argument list (in declaration + order) and the subset of those args declared with the ``optional`` + attribute in the subroutine body. + + Only the first definition of each subroutine name is recorded + (Fortran does not allow overloading at the subroutine level). + Subroutine and argument names are lowercased for case-insensitive + comparison. + + Optional-attribute detection scans body lines for type-declaration + lines of the form ``, ..., optional, ... :: [, ]...`` + while a tracking session is open for that subroutine. Optional + declarations inside a *nested* subroutine with the same name as an + outer subroutine are attributed to the inner sub (which is then + discarded by the first-occurrence-wins rule); optional declarations + inside a nested sub with a *different* name are correctly + attributed to that nested sub. + + Examples + -------- + >>> src = 'subroutine foo(a, b, c)\\n integer, intent(in) :: a, b, c\\nend subroutine foo\\n' + >>> result = _parse_subroutines(src) + >>> result['foo'].args + ['a', 'b', 'c'] + >>> sorted(result['foo'].optional) + [] + >>> src2 = 'subroutine bar()\\nend subroutine bar\\n' + >>> _parse_subroutines(src2)['bar'].args + [] + >>> src3 = ('subroutine baz(x, y, z)\\n' + ... ' integer, intent(in) :: x\\n' + ... ' integer, optional, intent(in) :: y\\n' + ... ' integer, intent(out), optional :: z\\n' + ... 'end subroutine baz\\n') + >>> sig = _parse_subroutines(src3)['baz'] + >>> sig.args + ['x', 'y', 'z'] + >>> sorted(sig.optional) + ['y', 'z'] + >>> sig.attrs['x'].intent, sig.attrs['x'].type_ + ('in', 'integer') + >>> sig.attrs['y'].optional + True + """ + logical = _join_continuation( + source.splitlines(keepends=True), filename=filename, + ) + args_by_name: Dict[str, List[str]] = {} + optional_by_name: Dict[str, Set[str]] = {} + attrs_by_name: Dict[str, Dict[str, _ArgAttrs]] = {} + # Stack of names whose body we are currently scanning. Each entry is + # the recorded name (for which we collect attrs) or ``None`` when + # this is a duplicate-name sub whose body should be skipped (its + # args were already discarded). + stack: List[Optional[str]] = [] + + for line in logical: + m = _SUB_RE.match(line) + if m: + name = m.group(1).lower() + arglist_raw = m.group(2) or '' + args = [a.strip().lower() for a in arglist_raw.split(',') + if a.strip()] + if name not in args_by_name: + args_by_name[name] = args + optional_by_name[name] = set() + attrs_by_name[name] = {} + stack.append(name) + else: + stack.append(None) # duplicate: ignore + continue + if _END_SUB_RE.match(line): + if stack: + stack.pop() + continue + if stack and stack[-1] is not None: + tracked = stack[-1] + arg_set = set(args_by_name[tracked]) + for var_name, attrs in _parse_decl_line(line).items(): + if var_name not in arg_set: + continue + # First decl line wins (Fortran disallows redeclaration, + # so this only matters for malformed input). + if var_name not in attrs_by_name[tracked]: + attrs_by_name[tracked][var_name] = attrs + if attrs.optional: + optional_by_name[tracked].add(var_name) + + return { + name: _SubSig(args=args_by_name[name], + optional=optional_by_name[name], + attrs=attrs_by_name[name]) + for name in args_by_name + } + + +def _load_source_tree(source_files: List[str]) -> Dict[str, _SubSig]: + """Read all Fortran source files and return a merged subroutine dict. + + Parameters + ---------- + source_files : list of str + Paths to ``.F90`` / ``.f90`` files. + + Returns + ------- + dict + Merged ``{subroutine_name_lower: _SubSig}``; first occurrence + wins if the same name appears in multiple files. + """ + merged: Dict[str, _SubSig] = {} + for fpath in source_files: + with open(fpath) as fh: + src = fh.read() + for name, sig in _parse_subroutines(src, filename=fpath).items(): + if name not in merged: + merged[name] = sig + return merged + + +# --------------------------------------------------------------------------- +# Module-level and derived-type parsing (for host / DDT validation) +# --------------------------------------------------------------------------- + +# Matches the start of a module definition (case-insensitive). Excludes +# ``module procedure`` so we don't mistake interface-block lines for +# module headers. +_MODULE_RE = re.compile(r'(?i)^\s*module\s+(?!procedure\b)(\w+)\s*$') +_END_MODULE_RE = re.compile(r'(?i)^\s*end\s*module\b') + +# Matches the start of a derived-type definition. Accepts the modern +# ``type :: name`` form, the older ``type, :: name`` form, and +# the bare ``type name`` form. Excludes ``type(x) ::`` declarations +# (those are variable decls of a type) by requiring no opening paren +# before the name on the type-defining form. +_TYPE_DEF_RE = re.compile( + r'(?i)^\s*type(?:\s*,\s*[^:]+)?\s*::\s*(\w+)\s*$' +) +_TYPE_DEF_BARE_RE = re.compile(r'(?i)^\s*type\s+(\w+)\s*$') +_END_TYPE_RE = re.compile(r'(?i)^\s*end\s*type\b') + +# Matches ``contains`` at module / type-block scope, used to recognise the +# boundary between module-level decls and the module's subroutines (we +# stop collecting module-level vars at the first ``contains``). Also +# applies inside a derived type with type-bound procedures. +_CONTAINS_RE = re.compile(r'(?i)^\s*contains\s*$') + + +class _ModuleSig(NamedTuple): + """Parsed module-level declarations from one Fortran module. + + Attributes + ---------- + vars : dict + ``{lower_name: _ArgAttrs}`` for every module-level variable + declaration above the ``contains`` line (or end of module). + Used to validate ``type = host`` table entries. + ddts : dict + ``{lower_type_name: {lower_component_name: _ArgAttrs}}`` for + every ``type :: X ... end type X`` block at module scope. + Used to validate ``type = ddt`` table entries. + """ + vars: Dict[str, '_ArgAttrs'] + ddts: Dict[str, Dict[str, '_ArgAttrs']] + + +def _parse_modules( + source: str, + filename: Optional[str] = None, +) -> Dict[str, _ModuleSig]: + """Extract module-level variable decls and derived-type definitions. + + Walks *source* line by line tracking three nested contexts: module, + derived-type block, and subroutine body. Module-level variable + declarations are collected only at module scope above ``contains``; + derived-type components are collected only inside a + ``type :: X ... end type X`` block; subroutine local decls are + ignored. + + The first definition of each module / type wins (Fortran does not + permit redefinition; this only matters for malformed input). + + Examples + -------- + >>> src = ('module my_mod\\n' + ... ' use kinds, only: kind_phys\\n' + ... ' integer :: nlev\\n' + ... ' real(kind=kind_phys) :: cp\\n' + ... ' type :: phys_t\\n' + ... ' real(kind=kind_phys) :: tk(:,:)\\n' + ... ' integer :: nlay\\n' + ... ' end type phys_t\\n' + ... 'contains\\n' + ... ' subroutine helper(x)\\n' + ... ' integer, intent(in) :: x\\n' + ... ' real :: local\\n' + ... ' end subroutine helper\\n' + ... 'end module my_mod\\n') + >>> mods = _parse_modules(src) + >>> sorted(mods.keys()) + ['my_mod'] + >>> sorted(mods['my_mod'].vars.keys()) + ['cp', 'nlev'] + >>> mods['my_mod'].vars['cp'].type_ + 'real' + >>> mods['my_mod'].vars['cp'].kind_ + 'kind_phys' + >>> sorted(mods['my_mod'].ddts.keys()) + ['phys_t'] + >>> sorted(mods['my_mod'].ddts['phys_t'].keys()) + ['nlay', 'tk'] + >>> mods['my_mod'].ddts['phys_t']['tk'].rank + 2 + >>> 'local' in mods['my_mod'].vars + False + """ + logical = _join_continuation( + source.splitlines(keepends=True), filename=filename, + ) + + modules: Dict[str, _ModuleSig] = {} + # Active module context (None outside of any module). + cur_mod_name: Optional[str] = None + cur_mod_vars: Dict[str, _ArgAttrs] = {} + cur_mod_ddts: Dict[str, Dict[str, _ArgAttrs]] = {} + # Active derived-type block context inside the current module. + cur_type_name: Optional[str] = None + cur_type_comps: Dict[str, _ArgAttrs] = {} + # Depth of subroutine / function nesting inside the current module. + # Decls inside a subroutine are local variables, not module-level + # state, and must be skipped. + sub_depth: int = 0 + # Once a ``contains`` is seen at module scope, module-level decls + # are done — the rest of the module is type-bound and subroutine + # bodies. We still need to track sub_depth to find the matching + # ``end module``. + past_module_contains: bool = False + + for line in logical: + # Module header / footer. + m = _MODULE_RE.match(line) + if m and cur_mod_name is None: + cur_mod_name = m.group(1).lower() + cur_mod_vars = {} + cur_mod_ddts = {} + cur_type_name = None + cur_type_comps = {} + sub_depth = 0 + past_module_contains = False + continue + if _END_MODULE_RE.match(line) and cur_mod_name is not None: + if cur_mod_name not in modules: + modules[cur_mod_name] = _ModuleSig( + vars=cur_mod_vars, ddts=cur_mod_ddts, + ) + cur_mod_name = None + continue + if cur_mod_name is None: + # Free-floating decls outside any module are not part of the + # host-validation surface. CCPP host code conventionally + # lives inside modules. + continue + + # Track subroutine / function nesting so we can skip local + # declarations. Use _SUB_RE for subroutines; functions are + # less common in CCPP host code but handled symmetrically. + if _SUB_RE.match(line) or re.match(r'(?i)\s*(?:(?:recursive|pure|elemental|impure)\s+)*(?:(?:real|integer|logical|complex|character|double\s*precision|type\s*\([^)]+\))\s+)?function\s+\w+', line): + sub_depth += 1 + continue + if _END_SUB_RE.match(line) or re.match(r'(?i)^\s*end\s*function\b', line): + if sub_depth > 0: + sub_depth -= 1 + continue + if sub_depth > 0: + continue + + # Derived-type block boundaries (only honoured at module scope, + # never inside a subroutine body). + if cur_type_name is None: + m = _TYPE_DEF_RE.match(line) or _TYPE_DEF_BARE_RE.match(line) + if m: + cur_type_name = m.group(1).lower() + cur_type_comps = {} + continue + else: + if _END_TYPE_RE.match(line): + if cur_type_name not in cur_mod_ddts: + cur_mod_ddts[cur_type_name] = cur_type_comps + cur_type_name = None + cur_type_comps = {} + continue + if _CONTAINS_RE.match(line): + # Type-bound procedures follow; no more components. + continue + # Inside a type block: every parsed decl is a component. + for name, attrs in _parse_decl_line(line).items(): + if name not in cur_type_comps: + cur_type_comps[name] = attrs + continue + + # Module scope: check for ``contains`` boundary and otherwise + # collect module-level variable decls. + if _CONTAINS_RE.match(line): + past_module_contains = True + continue + if past_module_contains: + continue + for name, attrs in _parse_decl_line(line).items(): + if name not in cur_mod_vars: + cur_mod_vars[name] = attrs + + return modules + + +def _load_modules_tree( + source_files: List[str], +) -> Tuple[Dict[str, _ModuleSig], Dict[str, Dict[str, _ArgAttrs]]]: + """Read all Fortran source files and return module + global DDT dicts. + + Returns a tuple ``(modules, ddt_index)``: + + * ``modules`` — ``{module_name_lower: _ModuleSig}``. First occurrence + wins if the same module name appears in multiple files. + * ``ddt_index`` — flat ``{type_name_lower: {component_name_lower: + _ArgAttrs}}`` mapping derived-type names to their component dicts, + collected across every parsed module. Used to resolve + ``type(name) :: var`` declarations whose underlying type lives in + a different module than the variable. First occurrence wins. + + Parameters + ---------- + source_files : list of str + Paths to ``.F90`` / ``.f90`` files. + """ + modules: Dict[str, _ModuleSig] = {} + ddt_index: Dict[str, Dict[str, _ArgAttrs]] = {} + for fpath in source_files: + with open(fpath) as fh: + src = fh.read() + for name, sig in _parse_modules(src, filename=fpath).items(): + if name not in modules: + modules[name] = sig + for ddt_name, comps in sig.ddts.items(): + if ddt_name not in ddt_index: + ddt_index[ddt_name] = comps + return modules, ddt_index + + +# --------------------------------------------------------------------------- +# Validation logic +# --------------------------------------------------------------------------- + +def _validate_scheme( + scheme_name: str, + scheme_store: SchemeStore, + subroutine_tree: Dict[str, _SubSig], + logger: logging.Logger, +) -> List[str]: + """Validate one scheme against *subroutine_tree*. + + Optional Fortran-only args (declared with the ``optional`` attribute + in the body and absent from the scheme metadata) are silently allowed + — they will never be passed at the cap call site, so the host need + not declare or provide them. Non-optional Fortran args missing from + the metadata, and metadata args missing from Fortran, remain hard + errors. + + Parameters + ---------- + scheme_name : str + scheme_store : SchemeStore + subroutine_tree : dict + logger : Logger + + Returns + ------- + list of str + Error messages (empty if all checks passed). + """ + errors: List[str] = [] + for phase in scheme_store.phases_for(scheme_name): + sub_name = '{}_{}'.format(scheme_name, phase).lower() + meta_vars = scheme_store.variables_for(scheme_name, phase) or [] + + logger.debug("Checking %s (phase=%s, sub=%s)", scheme_name, phase, sub_name) + + if sub_name not in subroutine_tree: + errors.append( + "Subroutine '{}' declared in metadata (scheme '{}', phase '{}') " + "not found in any source file.".format(sub_name, scheme_name, phase) + ) + continue + + sig = subroutine_tree[sub_name] + fort_args: List[str] = sig.args + fort_optional: Set[str] = sig.optional + meta_local_names: List[str] = [v.local_name.lower() for v in meta_vars] + + meta_set: Set[str] = set(meta_local_names) + fort_set: Set[str] = set(fort_args) + # Optional Fortran args that are absent from the metadata are + # silently allowed — never passed at the call site. + fort_only_optional: Set[str] = (fort_set - meta_set) & fort_optional + # Effective Fortran arg count for the count-mismatch check + # excludes those silent optional-only-in-Fortran args. + effective_fort_count = len(fort_args) - len(fort_only_optional) + + if len(meta_local_names) != effective_fort_count: + extra = '' + if fort_only_optional: + extra = ' (plus {} optional-only-in-Fortran args silently ' \ + 'allowed: {})'.format( + len(fort_only_optional), + sorted(fort_only_optional), + ) + # Degenerate-parse hint: if the Fortran subroutine was + # found but yielded zero args while metadata declares + # many, the parser almost certainly failed on the + # signature — most often because the file uses a + # continuation style (or other Fortran dialect feature) + # not handled by ``_join_continuation``. Flag it so the + # error trace points at the actual cause instead of a + # spurious "every metadata arg is missing" diff. + if len(fort_args) == 0 and len(meta_local_names) > 0: + extra += ( + " HINT: the Fortran signature parser found the " + "subroutine but extracted zero arguments. This is " + "almost always a parser bug, not a real mismatch — " + "common causes are unsupported continuation styles " + "or unusual signature syntax. Check the .F90 file " + "for the subroutine declaration and report a " + "validator bug if the signature looks normal." + ) + errors.append( + "Argument count mismatch for '{}': " + "metadata declares {} args {}, " + "Fortran declares {} required args.{}".format( + sub_name, + len(meta_local_names), meta_local_names, + effective_fort_count, extra, + ) + ) + + only_meta = meta_set - fort_set + only_fort_required = (fort_set - meta_set) - fort_optional + if only_meta: + errors.append( + "Arguments in metadata but not Fortran for '{}': {}".format( + sub_name, sorted(only_meta) + ) + ) + if only_fort_required: + errors.append( + "Non-optional arguments in Fortran but not metadata for '{}': {}".format( + sub_name, sorted(only_fort_required) + ) + ) + # Per-arg attribute checks for args present in BOTH sides. + meta_by_name = {v.local_name.lower(): v for v in meta_vars} + for name in sorted(meta_set & fort_set): + fattrs = sig.attrs.get(name) + if fattrs is None: + # Decl line failed to parse; skip attribute checks for + # this arg. Name-set check already covered presence. + continue + mvar = meta_by_name[name] + errors.extend( + _check_arg_attributes(sub_name, name, mvar, fattrs) + ) + # Optional flag — asymmetric: + # - metadata says optional, Fortran doesn't → hard error + # (cap may pass a missing arg, but Fortran requires it). + # - Fortran says optional, metadata doesn't → warning + # (cap always passes it; that's a valid subset of the + # Fortran contract, but the metadata writer may have + # intended to mark it optional). + if mvar.optional and not fattrs.optional: + errors.append( + "Arg '{}' on '{}': metadata declares optional=True " + "but Fortran does not carry the 'optional' attribute " + "(cap-side present() checks would be invalid)".format( + name, sub_name, + ) + ) + elif fattrs.optional and not mvar.optional: + logger.warning( + "Fortran argument '%s' on subroutine '%s' is " + "declared optional but metadata does not mark it " + "optional; cap will always pass it", + name, sub_name, + ) + # Fortran-only optional args (absent from metadata entirely): + # silently allowed but worth a heads-up — the host won't see + # them and the metadata writer may have meant to declare them. + for name in sorted(fort_only_optional): + logger.warning( + "Optional Fortran argument '%s' on subroutine '%s' is " + "absent from metadata; it will never be passed at the " + "call site", + name, sub_name, + ) + return errors + + +_EXTERNAL_TYPE_PREFIX_RE = re.compile(r'(?i)^external\s*:\s*[^:]+\s*:\s*') +_DDT_WRAPPER_RE = re.compile(r'(?i)^(?:type|class)\s*\(\s*(.+?)\s*\)\s*$') + + +def _normalize_type_for_comparison(type_str: str) -> str: + """Return a comparison-friendly form of a CCPP type string. + + Rules: + + * Lowercase, whitespace collapsed. + * ``type(name)`` / ``class(name)`` wrapper → bare ``name``. + * ``external::`` → bare ``typename`` (the module + part is metadata-only; Fortran uses the bare type name once the + module is brought in via a ``use`` clause). + * ``doubleprecision`` → ``double precision``. + * Anything else is returned as-is (intrinsics, DDT names, etc.). + + With this normalisation, a metadata declaration ``type = ty_rad_lw`` + matches a Fortran ``type(ty_rad_lw)`` dummy, and a metadata + ``type = external:mpi_f08:mpi_comm`` matches a Fortran + ``type(mpi_comm)`` dummy. Intrinsic comparisons (``real`` vs + ``real``) are unaffected. + + Examples + -------- + >>> _normalize_type_for_comparison('real') + 'real' + >>> _normalize_type_for_comparison('REAL') + 'real' + >>> _normalize_type_for_comparison('double precision') + 'double precision' + >>> _normalize_type_for_comparison('doubleprecision') + 'double precision' + >>> _normalize_type_for_comparison('ty_rad_lw') + 'ty_rad_lw' + >>> _normalize_type_for_comparison('type(ty_rad_lw)') + 'ty_rad_lw' + >>> _normalize_type_for_comparison('Type( Ty_Rad_LW )') + 'ty_rad_lw' + >>> _normalize_type_for_comparison('class(ty_rad_lw)') + 'ty_rad_lw' + >>> _normalize_type_for_comparison('external:mpi_f08:mpi_comm') + 'mpi_comm' + >>> _normalize_type_for_comparison('external : esmf_mod : esmf_clock') + 'esmf_clock' + """ + s = type_str.strip().lower() + s = re.sub(r'\s+', ' ', s) + s = _EXTERNAL_TYPE_PREFIX_RE.sub('', s) + m = _DDT_WRAPPER_RE.match(s) + if m: + s = m.group(1).strip() + if s == 'doubleprecision': + s = 'double precision' + return s + + +def _check_arg_attributes( + sub_name: str, + arg_name: str, + meta_var, + fort: _ArgAttrs, +) -> List[str]: + """Compare per-attribute consistency for one dummy argument. + + Compared attributes: ``intent``, ``type``, ``kind``, dimension + *rank* (number of dims). ``character`` length must be declared + CONSISTENTLY: the metadata mirrors the Fortran exactly -- ``len=*`` + matches only ``len=*`` and ``len=N`` only the identical ``len=N`` (no + wildcarding). The ``optional`` attribute is checked at the call site + in :func:`_validate_scheme` because one direction emits a warning + (logger-dependent) rather than an error. + + Returns a list of error message strings (empty on full match). + """ + errs: List[str] = [] + prefix = "Arg '{}' on '{}': ".format(arg_name, sub_name) + + # intent — only check when the metadata actually declares one (it's + # required for scheme vars but we don't reach this helper for + # non-scheme tables anyway). + meta_intent = (meta_var.intent or '').lower() + if meta_intent and fort.intent and meta_intent != fort.intent: + errs.append( + prefix + "intent mismatch (metadata={!r}, Fortran={!r})".format( + meta_intent, fort.intent, + ) + ) + elif meta_intent and not fort.intent: + errs.append( + prefix + "intent declared as {!r} in metadata but absent " + "from Fortran declaration".format(meta_intent) + ) + + # type — case-insensitive, with normalisation that puts intrinsic, + # DDT, and external types on equal footing. See + # :func:`_normalize_type_for_comparison` for the rules. + meta_type = (meta_var.type or '').strip() + if meta_type and fort.type_: + meta_norm = _normalize_type_for_comparison(meta_type) + fort_norm = _normalize_type_for_comparison(fort.type_) + if meta_norm != fort_norm: + errs.append( + prefix + "type mismatch (metadata={!r}, Fortran={!r})".format( + meta_var.type, fort.type_, + ) + ) + + # kind — case-insensitive. Empty matches empty. character has + # the ``len=*`` wildcard on either side. + meta_kind = (meta_var.kind or '').strip().lower() + fort_kind = fort.kind_ + if meta_type == 'character' or fort.type_ == 'character': + # Character length must be CONSISTENT between metadata and Fortran: + # the metadata mirrors the declaration, it does not loosely match it. + # ``len=*`` matches only ``len=*`` (assumed length on one side vs a + # concrete length on the other is a real inconsistency), and ``len=N`` + # matches only the identical ``len=N``. + if meta_kind != fort_kind: + errs.append( + prefix + "character length mismatch " + "(metadata={!r}, Fortran={!r}); the metadata kind must mirror " + "the Fortran declaration exactly -- len=* only matches len=*, " + "len=N only matches the same len=N".format(meta_kind, fort_kind) + ) + else: + if meta_kind != fort_kind: + errs.append( + prefix + "kind mismatch (metadata={!r}, Fortran={!r})".format( + meta_kind or '', fort_kind or '', + ) + ) + + # rank — number of dimensions. When the metadata ``local_name`` + # carries a subscript (sliced array entry such as + # ``q(:,:,index_of_water_vapor)``), the metadata's ``dimensions`` + # list describes the *view* after slicing, not the underlying + # Fortran rank. ``_expected_fort_rank`` resolves this: it returns + # the subscript width when a subscript is present, otherwise + # ``len(meta_var.dimensions)``. + meta_dims = list(meta_var.dimensions or []) + expected_rank = _expected_fort_rank(meta_var.local_name, meta_dims) + if expected_rank != fort.rank: + errs.append( + prefix + "rank mismatch (metadata implies Fortran rank {} " + "from local_name '{}' and dimensions {}, Fortran declares " + "rank {})".format( + expected_rank, meta_var.local_name, meta_dims, fort.rank, + ) + ) + + return errs + + +def _base_local_name(local_name: str) -> str: + """Return the bare Fortran identifier from a metadata ``local_name``. + + Host / DDT metadata occasionally carries subscripted ``local_name`` + values (sliced array entries) — the matching Fortran decl carries the + bare identifier, so strip the subscript before lookup. Lowercase for + case-insensitive comparison. + + Examples + -------- + >>> _base_local_name('cp') + 'cp' + >>> _base_local_name('Phys_State') + 'phys_state' + >>> _base_local_name('tk(:,:)') + 'tk' + >>> _base_local_name('dqdt(:,:,index_of_cloud)') + 'dqdt' + """ + name = local_name.strip() + if '(' in name: + name = name.split('(', 1)[0] + return name.lower() + + +def _expected_fort_rank(local_name: str, meta_dims: List[str]) -> int: + """Return the Fortran rank implied by a metadata ``local_name``. + + Sliced metadata local names (``q(:,:,index_of_X)``) express a + reduced-rank *view* of a higher-rank Fortran component: every + subscript entry consumes one dimension of the underlying Fortran + variable, but only ``:`` entries survive into the resulting view's + rank. The metadata's ``dimensions =`` list describes the view, not + the underlying variable — so the expected Fortran rank equals the + total number of subscript entries, not ``len(meta_dims)``. + + For bare local names (no subscript), Fortran rank simply equals the + metadata-declared dimension count. + + Examples + -------- + >>> _expected_fort_rank('cp', []) + 0 + >>> _expected_fort_rank('tk', ['horizontal_dimension', + ... 'vertical_layer_dimension']) + 2 + >>> _expected_fort_rank('q(:,:,index_of_water_vapor)', + ... ['horizontal_dimension', + ... 'vertical_layer_dimension']) + 3 + >>> _expected_fort_rank('q(:)', ['horizontal_dimension']) + 1 + >>> _expected_fort_rank('q(:,:,:)', ['horizontal_dimension', + ... 'vertical_layer_dimension', + ... 'number_of_tracers']) + 3 + """ + name = local_name.strip() + if '(' not in name: + return len(meta_dims) + inner = name.split('(', 1)[1] + inner = inner.rsplit(')', 1)[0] + # Each subscript entry consumes one Fortran dimension; paren-aware + # split protects nested expressions like ``my(:,foo(a,b),:)`` from + # being miscounted (none of the in-tree fixtures use that today, but + # the parser permits it). + return len(_paren_aware_split(inner, ',')) + + +def _validate_host_table( + table, + modules_tree: Dict[str, _ModuleSig], + logger: logging.Logger, +) -> List[str]: + """Validate a ``type = host`` metadata table against its Fortran module. + + The Fortran module name is taken from ``table.module_name`` when set, + otherwise from ``table.table_name`` (the .meta convention). For each + metadata variable, look up the matching module-level decl by base + local-name and reuse :func:`_check_arg_attributes` for the per-attr + checks (intent is silently ignored since host vars carry no intent). + + Returns a list of error message strings (empty on full match). When + the named module is missing entirely, a single "module not found" + error is returned and per-variable checks are skipped. + """ + errors: List[str] = [] + mod_name = (table.module_name or table.table_name).lower() + sig = modules_tree.get(mod_name) + if sig is None: + errors.append( + "Host module '{}' (from table '{}' in '{}') not found in any " + "source file.".format(mod_name, table.table_name, table.file_path) + ) + return errors + + logger.debug( + "Checking host table '%s' against module '%s' (%d module-level vars)", + table.table_name, mod_name, len(sig.vars), + ) + + for mvar in table.variables(): + base = _base_local_name(mvar.local_name) + fattrs = sig.vars.get(base) + if fattrs is None: + errors.append( + "Host variable '{}' (standard_name '{}') declared in " + "metadata table '{}' not found as a module-level " + "declaration in Fortran module '{}'.".format( + mvar.local_name, mvar.standard_name, + table.table_name, mod_name, + ) + ) + continue + errors.extend( + _check_arg_attributes(mod_name, base, mvar, fattrs) + ) + return errors + + +def _validate_ddt_table( + table, + ddt_index: Dict[str, Dict[str, _ArgAttrs]], + logger: logging.Logger, +) -> List[str]: + """Validate a ``type = ddt`` metadata table against its Fortran type. + + The DDT name is ``table.table_name`` (the .meta convention: the table + name is the Fortran type name). The matching ``type :: X ... end + type X`` block may live in any parsed module — looked up via the flat + *ddt_index* built by :func:`_load_modules_tree`. For each metadata + component, match against the type block by base local-name and reuse + :func:`_check_arg_attributes` for per-attr checks. + + Returns a list of error message strings (empty on full match). + """ + errors: List[str] = [] + ddt_name = table.table_name.lower() + comps = ddt_index.get(ddt_name) + if comps is None: + errors.append( + "DDT '{}' (table in '{}') not found as a derived-type definition " + "in any source file.".format(ddt_name, table.file_path) + ) + return errors + + logger.debug( + "Checking DDT table '%s' (%d components in Fortran)", + ddt_name, len(comps), + ) + + for mvar in table.variables(): + base = _base_local_name(mvar.local_name) + fattrs = comps.get(base) + if fattrs is None: + errors.append( + "DDT component '{}' (standard_name '{}') declared in " + "metadata table '{}' not found as a component of Fortran " + "type '{}'.".format( + mvar.local_name, mvar.standard_name, + table.table_name, ddt_name, + ) + ) + continue + errors.extend( + _check_arg_attributes(ddt_name, base, mvar, fattrs) + ) + return errors + + +def _check_definition_character_lengths(table) -> List[str]: + """Reject ``character ... kind = len=*`` in a definition-site table. + + Host and DDT metadata *define* the storage for their character + variables (a host module variable / a derived-type component), so an + assumed length (``len=*``) is illegal there: it is valid only on a + dummy argument (a scheme arg, or a control/lifecycle variable) where + the storage is supplied by the caller. Apply this only to ``host`` / + ``ddt`` tables -- ``control`` tables are exempt. This mirrors the + generator's ``build_flat_host_dict`` guard and is checked independently + of the Fortran-vs-metadata comparison (a ``character(len=*)`` host decl + is invalid Fortran in its own right and would never be found). + + Returns a list of error message strings (empty when none offend). + """ + errors: List[str] = [] + for mvar in table.variables(): + if ((mvar.type or '').strip().lower() == 'character' + and (mvar.kind or '').strip().lower() == 'len=*'): + errors.append( + "Character variable '{}' (standard_name '{}') in {} table " + "'{}' declares kind='len=*'; host and DDT metadata must give " + "character variables a concrete length (e.g. kind=len=512) " + "-- assumed length is valid only for dummy arguments (scheme " + "args and control/lifecycle variables).".format( + mvar.local_name, mvar.standard_name, + table.table_type, table.table_name, + ) + ) + return errors + + +_FORTRAN_EXTENSIONS = ('.F90', '.f90', '.F', '.f') + + +def _fortran_file_for_table(table) -> Optional[str]: + """Return the Fortran source path for a scheme *table* using ``source_path``. + + The convention is that the ``.F90`` file has the same base name as the + ``.meta`` file but lives in the directory given by ``table.source_path`` + (which defaults to the ``.meta`` file's own directory when not set). + + Returns ``None`` if no matching file is found. + + Parameters + ---------- + table : MetadataTable + A scheme table with ``file_path`` and ``source_path`` set. + + Returns + ------- + str or None + Absolute path to the Fortran source file, or ``None``. + + Examples + -------- + >>> from metadata.parse_tools import ParseContext + >>> from metadata.metadata_table import MetadataTable + >>> import tempfile, os + >>> with tempfile.TemporaryDirectory() as d: + ... meta = os.path.join(d, 'foo.meta') + ... fort = os.path.join(d, 'foo.F90') + ... open(fort, 'w').close() + ... ctx = ParseContext(0, meta) + ... t = MetadataTable('foo', 'scheme', meta, ctx) + ... t.apply_table_props({}) + ... os.path.basename(_fortran_file_for_table(t)) + 'foo.F90' + """ + meta_base = os.path.splitext(os.path.basename(table.file_path))[0] + search_dir = table.source_path or os.path.dirname(os.path.abspath(table.file_path)) + for ext in _FORTRAN_EXTENSIONS: + candidate = os.path.join(search_dir, meta_base + ext) + if os.path.isfile(candidate): + return candidate + return None + + +def validate( + scheme_files: List[str], + source_files: Optional[List[str]] = None, + host_files: Optional[List[str]] = None, + logger: Optional[logging.Logger] = None, +) -> List[str]: + """Validate scheme + host metadata against Fortran source files. + + Scheme tables in *scheme_files* are validated against the subroutine + signatures in *source_files* (the existing scheme-side check). + ``type = host`` and ``type = ddt`` tables in *host_files* are + additionally validated against module-level decls and derived-type + definitions in those same source files. ``type = control`` tables + are silent-skipped (no Fortran source backs control vars — they are + framework-injected at the cap call sites). A ``type = scheme`` table + appearing in *host_files* is a hard error: schemes must be passed via + *scheme_files* so the validator can find the per-phase subroutines. + + When *source_files* is ``None`` or empty, the validator resolves the + Fortran source for each scheme / host / ddt table automatically using + the ``source_path`` attribute from the metadata (defaulting to the + ``.meta`` file's directory if ``source_path`` is absent). Pass an + explicit list to override. + + Parameters + ---------- + scheme_files : list of str + Paths to scheme ``.meta`` files. + source_files : list of str, optional + Explicit Fortran source files to scan. If omitted, auto-discovered + via ``source_path`` in the metadata. + host_files : list of str, optional + Paths to host ``.meta`` files (``type = host`` / ``type = ddt`` / + ``type = control``). Defaults to an empty list (scheme-only + validation). + logger : Logger, optional + + Returns + ------- + list of str + All validation error messages (empty means success). + + Raises + ------ + CCPPError + On metadata parse errors or missing files. + """ + log = logger or _LOGGER + + scheme_files = list(scheme_files or []) + host_files = list(host_files or []) + + # At least one of --scheme-files / --host-files must be supplied; + # otherwise the validator has nothing to do and would silently report + # "Validation passed." That was the old scheme-only behaviour with + # host metadata accidentally passed in; the new contract requires + # the caller to opt in to one side or the other (or both). + if not scheme_files and not host_files: + raise CCPPError( + "ccpp_validator requires at least one of --scheme-files or " + "--host-files; neither was supplied." + ) + + log.info("Loading scheme metadata from %d file(s)", len(scheme_files)) + scheme_tables = [] + for fpath in scheme_files: + scheme_tables.extend(parse_metadata_file(fpath)) + + # Reject host / control / suite tables passed via --scheme-files. + # ``type = ddt`` IS allowed alongside scheme tables — schemes + # routinely co-locate their own derived-type definitions in the + # same .meta file (e.g. radiation schemes defining their internal + # ty_rad_lw / ty_rad_sw DDTs). Such DDTs go through the same + # ``_validate_ddt_table`` pass as host-side DDTs. Symmetric to + # the rejection of scheme tables in --host-files (see below): each + # CLI flag has a single, narrow responsibility so misclassified + # host / control / suite .meta files fail fast with a clear pointer. + scheme_nonscheme_violations = [ + t for t in scheme_tables + if t.table_type in ('host', 'control', 'suite') + ] + if scheme_nonscheme_violations: + details = sorted({ + "{} (type = {})".format(t.table_name, t.table_type) + for t in scheme_nonscheme_violations + }) + raise CCPPError( + "Only type = scheme and type = ddt tables may appear in " + "--scheme-files; host / control / suite tables must be " + "passed via --host-files instead. Offending tables: {}".format( + details, + ) + ) + + scheme_store = SchemeStore.build_from(scheme_tables) + log.info("Found %d scheme(s): %s", len(scheme_store.scheme_names()), scheme_store.scheme_names()) + + # Collect DDT tables that travelled in via --scheme-files; they get + # the same per-component validation as host-side DDTs. + scheme_ddt_tables = [t for t in scheme_tables if t.table_type == 'ddt'] + + host_tables = [] + if host_files: + log.info("Loading host metadata from %d file(s)", len(host_files)) + for fpath in host_files: + host_tables.extend(parse_metadata_file(fpath)) + + # Reject scheme tables passed via --host-files: they wouldn't get + # phase-aware validation, and silent acceptance would hide a real + # mistake. Fail fast before any per-table check runs. + host_scheme_violations = [ + t for t in host_tables if t.is_scheme + ] + if host_scheme_violations: + names = sorted({t.table_name for t in host_scheme_violations}) + raise CCPPError( + "type = scheme tables may not appear in --host-files; pass " + "them via --scheme-files instead. Offending tables: {}".format( + names, + ) + ) + + if source_files: + log.info("Scanning %d explicit Fortran source file(s)", len(source_files)) + resolved_sources = list(source_files) + else: + # Auto-discover Fortran files via source_path in each scheme + + # host / ddt table (control tables have no Fortran source). + resolved_sources = [] + for tbl in scheme_tables: + if tbl.table_type == 'control': + continue + fort = _fortran_file_for_table(tbl) + if fort: + resolved_sources.append(fort) + log.debug("Resolved Fortran source for '%s': %s", tbl.table_name, fort) + else: + log.warning( + "No Fortran source found for %s '%s' (source_path='%s')", + tbl.table_type, tbl.table_name, tbl.source_path, + ) + for tbl in host_tables: + if tbl.table_type == 'control': + continue + fort = _fortran_file_for_table(tbl) + if fort: + resolved_sources.append(fort) + log.debug( + "Resolved Fortran source for host/ddt table '%s': %s", + tbl.table_name, fort, + ) + else: + log.warning( + "No Fortran source found for %s table '%s' " + "(source_path='%s')", + tbl.table_type, tbl.table_name, tbl.source_path, + ) + subroutine_tree = _load_source_tree(resolved_sources) + log.info("Found %d subroutine definitions", len(subroutine_tree)) + + modules_tree: Dict[str, _ModuleSig] = {} + ddt_index: Dict[str, Dict[str, _ArgAttrs]] = {} + # DDT validation needs the module/type parse whenever any DDT table is + # in play — scheme-co-located DDTs count too. + if host_tables or scheme_ddt_tables: + modules_tree, ddt_index = _load_modules_tree(resolved_sources) + log.info( + "Found %d module(s) and %d derived-type definition(s)", + len(modules_tree), len(ddt_index), + ) + + all_errors: List[str] = [] + for sname in scheme_store.scheme_names(): + all_errors.extend( + _validate_scheme(sname, scheme_store, subroutine_tree, log) + ) + # Scheme-co-located DDTs validate the same way as host-side DDTs; + # their character components are definition sites too (a DDT component + # may not be assumed-length), so the len=* guard applies. + for tbl in scheme_ddt_tables: + all_errors.extend(_check_definition_character_lengths(tbl)) + all_errors.extend(_validate_ddt_table(tbl, ddt_index, log)) + for tbl in host_tables: + # Host and DDT tables define their character storage: reject len=* + # regardless of the Fortran-comparison outcome. Control tables are + # exempt -- their character vars are pass-through dummy arguments. + if tbl.table_type == 'host': + all_errors.extend(_check_definition_character_lengths(tbl)) + all_errors.extend(_validate_host_table(tbl, modules_tree, log)) + elif tbl.table_type == 'ddt': + all_errors.extend(_check_definition_character_lengths(tbl)) + all_errors.extend(_validate_ddt_table(tbl, ddt_index, log)) + elif tbl.table_type == 'control': + log.info( + "Skipping control table '%s' (no Fortran source backs " + "control variables)", tbl.table_name, + ) + # scheme tables were already rejected above; suite tables aren't + # expected here but would be silent-skipped by omission. + + if all_errors: + log.warning("%d validation error(s) found.", len(all_errors)) + else: + log.info("Validation passed.") + return all_errors + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog='ccpp_validator.py', + description='Validate CCPP scheme Fortran source against metadata', + ) + parser.add_argument( + '--scheme-files', + required=False, + default='', + metavar='FILE[,FILE...]', + help=( + 'Comma-separated scheme metadata (.meta) files. May contain ' + 'type = scheme tables and type = ddt tables (schemes routinely ' + "co-locate their own DDTs in the same .meta file); host / " + 'control / suite tables are rejected (pass them via ' + '--host-files instead). At least one of --scheme-files or ' + '--host-files must be supplied.' + ), + ) + parser.add_argument( + '--host-files', + required=False, + default='', + metavar='FILE[,FILE...]', + help=( + 'Comma-separated host metadata (.meta) files. ' + 'type = host / type = ddt tables in these files are validated ' + 'against module-level decls and derived-type definitions in ' + 'the same --source-files. type = control is silent-skipped ' + '(no Fortran source backs control vars). type = scheme is ' + 'rejected (pass via --scheme-files instead).' + ), + ) + parser.add_argument( + '--source-files', + required=True, + metavar='FILE[,FILE...]', + help='Comma-separated Fortran source (.F90) files', + ) + parser.add_argument( + '--verbose', '-v', + action='count', + default=0, + help='Increase verbosity (use twice for DEBUG)', + ) + # legacy-compat: transient migration shim (delete the argument, + # the enable() call below, and the rest of the legacy_compat + # touchpoints when the migration is complete). + parser.add_argument( + '--legacy-mode', + action='store_true', + help=( + "TRANSIENT MIGRATION SHIM. Accept legacy CCPP standard " + "names (currently 'horizontal_loop_extent') in scheme " + "metadata and silently rewrite them to their canonical " + "capgen equivalents ('horizontal_dimension'). Emits a " + "loud warning at startup. Will be removed." + ), + ) + # NB: --gfs-dim-aliases is NOT exposed here. That shim only takes + # effect inside generator.suite_resolver._canonical_dim, which the + # validator never invokes (the validator compares metadata against + # Fortran source, not host metadata against scheme metadata), so + # the flag would be a no-op. See capgen/ccpp_capgen.py. + # auto-clone-constituents: transient legacy shim. This one DOES + # belong on the validator because the shim extends the parser's + # ``_KNOWN_ATTRS`` set — without the flag the validator rejects + # the four legacy attrs (default_value/min_value/water_species/ + # mixing_ratio_type) with "Unknown variable attribute", which + # blocks pre-flight validation runs. The validator never builds + # a host_dict, so the shim's single-instance host guard is not + # called here (it's a no-op without a host metadata pass). + parser.add_argument( + '--legacy-auto-clone-constituents', + action='store_true', + help=( + "TRANSIENT LEGACY SHIM. Accept four legacy constituent " + "attributes (default_value, min_value, water_species, " + "mixing_ratio_type) on scheme args. Mirrors the same " + "flag on ccpp_capgen so legacy scheme metadata that " + "needs auto-clone-static-constituent codegen can be " + "validated against its Fortran source. Emits a loud " + "warning at startup. Will be removed." + ), + ) + return parser + + +def main(argv: Optional[List[str]] = None) -> int: + parser = _build_parser() + args = parser.parse_args(argv) + + if args.verbose == 0: + set_log_level(_LOGGER, logging.WARNING) + elif args.verbose == 1: + set_log_level(_LOGGER, logging.INFO) + else: + set_log_level(_LOGGER, logging.DEBUG) + + # legacy-compat: transient migration shim. Emit the loud banner + # before any parsing happens so user has fair warning. + if args.legacy_mode: + from metadata import legacy_compat + legacy_compat.enable(_LOGGER) + + # auto-clone-constituents: transient legacy shim. Emit the loud + # banner before any parsing happens so user has fair warning. + # The single-instance host guard is intentionally NOT invoked + # here (no host metadata pass in the validator). + if args.legacy_auto_clone_constituents: + from metadata import auto_clone_constituents + auto_clone_constituents.enable(_LOGGER) + + scheme_files = [f.strip() for f in args.scheme_files.split(',') if f.strip()] + source_files = [f.strip() for f in args.source_files.split(',') if f.strip()] + host_files = [f.strip() for f in args.host_files.split(',') if f.strip()] + + try: + errors = validate(scheme_files, source_files, host_files=host_files) + except CCPPError as exc: + _LOGGER.error("%s", exc) + return 2 + except OSError as exc: + _LOGGER.error("File error: %s", exc) + return 2 + + if errors: + for err in errors: + print("ERROR:", err, file=sys.stderr) + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/capgen/generator/__init__.py b/capgen/generator/__init__.py new file mode 100644 index 00000000..9eabbc64 --- /dev/null +++ b/capgen/generator/__init__.py @@ -0,0 +1 @@ +"""Cap code generation for ccpp-capgen.""" diff --git a/capgen/generator/datatable.py b/capgen/generator/datatable.py new file mode 100644 index 00000000..97235ed9 --- /dev/null +++ b/capgen/generator/datatable.py @@ -0,0 +1,485 @@ +#!/usr/bin/env python3 + +"""Write ``datatable.xml`` — the generator database consumed by ``ccpp_datafile.py`` +and CMake to discover compilation targets, scheme modules, and suite structure. + +XML layout +---------- + +Two top-level file sections partition capgen's outputs by language: + +* ```` lists *Fortran sources only* — utilities, the host-facing + API, and per-suite cap modules. All paths here end in ``.F90`` and are + intended to be compiled by the host build system. +* ```` lists *non-Fortran* artifacts — generated + ``.meta`` files and expanded SDF XML. These are produced for debugging + and downstream tooling; they are *not* compiled. + +:: + + + + + + /abs/path/ccpp_kinds.F90 + + + /abs/path/_ccpp_cap.F90 + + + /abs/path/ccpp__cap.F90 + ... + + + + /abs/path/scheme_x.F90 + ... + + + + /abs/path/ccpp_.meta + ... + + + /abs/path/ccpp__expanded.xml + ... + + + + + + + + ... + + + + ... + + ... + + + + + + + ... + + + + + + + + + + + ... + + + + + + + + + + + + + + + ... + + + ... + + + + +Compatibility notes +------------------- +The ```` section uses the same phase element tag names as the new +generator's phase vocabulary (``init``, ``run``, ``timestep_init``, +``timestep_final``, ``final``, ``register``). ``ccpp_datafile.py`` iterates +scheme children by tag rather than filtering on specific names, so this is +forward-compatible. +""" + +import io +import logging +import os +import xml.etree.ElementTree as ET +from typing import Dict, List, Optional, Set, Tuple + +from metadata.parse_tools import write_if_changed +from generator.suite_resolver import SuiteResolution, iter_phase_calls + +_API_DICT_NAME = 'ccpp_api' + + +def _build_capgen_files( + root: ET.Element, + utility_paths: List[str], + host_file_paths: List[str], + suite_file_paths: List[str], +) -> None: + """Append ```` to *root*. + + ```` lists Fortran sources generated by capgen (utilities, + the host-facing API, and per-suite caps). Non-Fortran inspection + artifacts (e.g. ``.meta`` files, expanded SDFs) live in + ```` and are emitted by :func:`_build_inspection_files`. + + ``host_file_paths`` lists files generated for the host-facing API. In + capgen this is the static API (``_ccpp_cap.F90``); the section is + emitted unconditionally (possibly empty) to keep the schema stable. + """ + capgen_files = ET.SubElement(root, 'capgen_files') + + utilities = ET.SubElement(capgen_files, 'utilities') + for path in utility_paths: + f = ET.SubElement(utilities, 'file') + f.text = path + + host_files = ET.SubElement(capgen_files, 'host_files') + for path in host_file_paths: + f = ET.SubElement(host_files, 'file') + f.text = path + + suite_files = ET.SubElement(capgen_files, 'suite_files') + for path in suite_file_paths: + f = ET.SubElement(suite_files, 'file') + f.text = path + + +def _build_scheme_files( + root: ET.Element, + scheme_file_paths: List[str], +) -> None: + """Append ```` to *root*. + + Lists the Fortran source files of schemes actually referenced by some + loaded suite (group phase calls + suite-level ```` / ````). + These files are *not* generated by capgen — they are the + user-supplied scheme implementations the build system must compile. + The section is always written (possibly empty) so the schema stays + stable for ``ccpp_datafile.py`` consumers. + """ + scheme_files = ET.SubElement(root, 'scheme_files') + for path in scheme_file_paths: + f = ET.SubElement(scheme_files, 'file') + f.text = path + + +def _build_inspection_files( + root: ET.Element, + suite_meta_paths: Optional[List[str]] = None, + expanded_sdf_paths: Optional[List[str]] = None, +) -> None: + """Append ```` to *root*. + + Inspection artifacts are non-Fortran outputs that capgen emits for + debugging and downstream tooling. They are *not* compiled. Each kind + of artifact lives in its own subsection so consumers can pick a specific + file type or take them all together via ``--inspection-files``. + + Subsections written: + + * ```` — generated ``ccpp_.meta`` files (the + suite-data variable metadata table). + * ```` — fully expanded suite-definition XML + (``ccpp__expanded.xml``), with any ```` references + resolved. + + The section is always written (possibly with empty subsections) to keep + the schema stable. + """ + inspection = ET.SubElement(root, 'inspection_files') + + suite_meta_files = ET.SubElement(inspection, 'suite_meta_files') + for path in suite_meta_paths or []: + f = ET.SubElement(suite_meta_files, 'file') + f.text = path + + expanded_sdf_files = ET.SubElement(inspection, 'expanded_sdf_files') + for path in expanded_sdf_paths or []: + f = ET.SubElement(expanded_sdf_files, 'file') + f.text = path + + +def _build_schemes( + root: ET.Element, + suite_resolutions: List[SuiteResolution], + scheme_store, +) -> None: + """Append ```` to *root*. + + Each scheme appears once; its phases are the union of phases seen across + all suites. Duplicate scheme names (scheme used in multiple suites) are + deduplicated — the call_list is the same regardless of which suite uses + the scheme. + """ + schemes_elem = ET.SubElement(root, 'schemes') + seen_scheme_phases: Dict[str, Set[str]] = {} + + for suite_resolution in suite_resolutions: + for resolved_group in suite_resolution.groups: + for phase_name, items in resolved_group.phase_calls.items(): + for resolved_call in iter_phase_calls(items): + sname = resolved_call.scheme_name + if sname not in seen_scheme_phases: + seen_scheme_phases[sname] = set() + seen_scheme_phases[sname].add(phase_name) + # Suite-level / hooks are not part of any group's + # phase_calls — fold them in explicitly so the schemes they + # name appear in (and downstream queries pick them + # up). Their phase is fixed by the SDF element. + for resolved_call, phase_name in ( + (suite_resolution.suite_init_call, 'init'), + (suite_resolution.suite_final_call, 'final'), + ): + if resolved_call is None: + continue + seen_scheme_phases.setdefault(resolved_call.scheme_name, set()).add(phase_name) + + for sname in sorted(seen_scheme_phases): + scheme_elem = ET.SubElement(schemes_elem, 'scheme') + scheme_elem.set('name', sname) + for phase in sorted(seen_scheme_phases[sname]): + phase_elem = ET.SubElement(scheme_elem, phase) + phase_elem.set('name', sname) + phase_elem.set('subroutine_name', '{}_{}'.format(sname, phase)) + phase_elem.set('module', sname) + call_list = ET.SubElement(phase_elem, 'call_list') + mvars = scheme_store.variables_for(sname, phase) + if mvars: + for mv in mvars: + var = ET.SubElement(call_list, 'var') + var.set('name', mv.standard_name) + var.set('intent', mv.intent or '') + var.set('local_name', mv.local_name) + if mv.diagnostic_name: + var.set('diagnostic_name', mv.diagnostic_name) + if mv.diagnostic_name_fixed: + var.set('diagnostic_name_fixed', + mv.diagnostic_name_fixed) + + +def _build_api( + root: ET.Element, + suite_resolutions: List[SuiteResolution], +) -> None: + """Append ``...`` to *root*.""" + api_elem = ET.SubElement(root, 'api') + suites_elem = ET.SubElement(api_elem, 'suites') + + for suite_resolution in suite_resolutions: + suite_elem = ET.SubElement(suites_elem, 'suite') + suite_elem.set('name', suite_resolution.suite_name) + for resolved_group in suite_resolution.groups: + group_elem = ET.SubElement(suite_elem, 'group') + group_elem.set('name', resolved_group.group_name) + seen: Set[str] = set() + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + if resolved_call.scheme_name not in seen: + seen.add(resolved_call.scheme_name) + sch = ET.SubElement(group_elem, 'scheme') + sch.text = resolved_call.scheme_name + + +def _build_var_dictionaries( + root: ET.Element, + host_dict, + host_name: str, + suite_resolutions: List[SuiteResolution], +) -> None: + """Append ```` to *root*. + + Emits one dictionary entry per scope in the lookup chain + (``host`` → ``api`` → ``suite`` → ``group`` → ``group_call_list``). + The ``parent`` attribute on each entry preserves the inheritance chain + that ``ccpp_datafile.py`` walks when resolving ``protected`` and + ``--exclude-protected``. + + The host dictionary carries one ```` per entry in *host_dict* + (host model + control vars). Each group's call-list dictionary + carries one ```` per (standard_name, intent) tuple seen at any + phase call site in that group. + """ + if host_dict is None: + return + + dicts = ET.SubElement(root, 'var_dictionaries') + + host_d = ET.SubElement(dicts, 'var_dictionary') + host_d.set('name', host_name) + host_d.set('type', 'host') + h_vars = ET.SubElement(host_d, 'variables') + for std_name in sorted(host_dict): + entry = host_dict[std_name] + v = ET.SubElement(h_vars, 'var') + v.set('name', std_name) + if entry.local_name: + v.set('local_name', entry.local_name) + if getattr(entry, 'protected', False): + v.set('protected', 'True') + + api_d = ET.SubElement(dicts, 'var_dictionary') + api_d.set('name', _API_DICT_NAME) + api_d.set('type', 'api') + api_d.set('parent', host_name) + ET.SubElement(api_d, 'variables') + + for suite_resolution in suite_resolutions: + suite_d = ET.SubElement(dicts, 'var_dictionary') + suite_d.set('name', suite_resolution.suite_name) + suite_d.set('type', 'suite') + suite_d.set('parent', _API_DICT_NAME) + ET.SubElement(suite_d, 'variables') + + for resolved_group in suite_resolution.groups: + group_d = ET.SubElement(dicts, 'var_dictionary') + group_d.set('name', resolved_group.group_name) + group_d.set('type', 'group') + group_d.set('parent', suite_resolution.suite_name) + ET.SubElement(group_d, 'variables') + + call_d = ET.SubElement(dicts, 'var_dictionary') + call_d.set('name', '{}_call_list'.format(resolved_group.group_name)) + call_d.set('type', 'group_call_list') + call_d.set('parent', resolved_group.group_name) + cl_vars = ET.SubElement(call_d, 'variables') + + seen: Set[Tuple[str, str]] = set() + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: + intent = arg.intent or '' + key = (arg.standard_name, intent) + if key in seen: + continue + seen.add(key) + v = ET.SubElement(cl_vars, 'var') + v.set('name', arg.standard_name) + if intent: + v.set('intent', intent) + if arg.scheme_local_name: + v.set('local_name', arg.scheme_local_name) + + +def write_datatable( + suite_resolutions: List[SuiteResolution], + scheme_store, + utility_paths: List[str], + suite_file_paths: List[str], + output_root: str, + host_file_paths: Optional[List[str]] = None, + scheme_file_paths: Optional[List[str]] = None, + dependency_paths: Optional[List[str]] = None, + suite_meta_paths: Optional[List[str]] = None, + expanded_sdf_paths: Optional[List[str]] = None, + host_dict=None, + host_name: str = 'host', + logger: Optional[logging.Logger] = None, +) -> str: + """Write ``datatable.xml`` and return its absolute path. + + Parameters + ---------- + suite_resolutions : list of SuiteResolution + scheme_store : SchemeStore + utility_paths : list of str + Absolute paths to utility Fortran files (e.g. ``ccpp_kinds.F90``). + suite_file_paths : list of str + Absolute paths to generated suite cap files. + output_root : str + Output directory. + host_file_paths : list of str, optional + Absolute paths to host-facing API files (capgen emits + ``_ccpp_cap.F90`` here). The ```` section is + always written (possibly empty). + scheme_file_paths : list of str, optional + Absolute paths to the Fortran source files (``.F90`` / ``.F`` / ...) + of schemes actually referenced by the loaded suites. Written as + ```` children of the ```` element so build + systems can compile exactly the scheme set the suites consume. + dependency_paths : list of str, optional + Absolute paths of scheme dependency files (collected from + ``MetadataTable.dependencies``). Written as ```` + children of the ```` element. + suite_meta_paths : list of str, optional + Absolute paths to generated ``ccpp_.meta`` files (inspection + artifacts; written under ````). + expanded_sdf_paths : list of str, optional + Absolute paths to generated ``ccpp__expanded.xml`` files + (inspection artifacts; written under + ````). + + Returns + ------- + str + Absolute path to the written ``datatable.xml``. + + Examples + -------- + >>> import tempfile, os + >>> from generator.datatable import write_datatable + >>> from unittest.mock import MagicMock + >>> suite_resolution = MagicMock() + >>> suite_resolution.suite_name = 'test' + >>> suite_resolution.groups = [] + >>> suite_resolution.suite_init_call = None + >>> suite_resolution.suite_final_call = None + >>> store = MagicMock() + >>> with tempfile.TemporaryDirectory() as d: + ... path = write_datatable([suite_resolution], store, [], [], d) + ... os.path.basename(path) + 'datatable.xml' + """ + os.makedirs(output_root, exist_ok=True) + + root = ET.Element('ccpp_datatable') + root.set('version', '1.0') + + _build_capgen_files( + root, utility_paths, host_file_paths or [], + suite_file_paths, + ) + _build_scheme_files(root, scheme_file_paths or []) + _build_inspection_files( + root, + suite_meta_paths=suite_meta_paths, + expanded_sdf_paths=expanded_sdf_paths, + ) + _build_schemes(root, suite_resolutions, scheme_store) + _build_api(root, suite_resolutions) + _build_var_dictionaries(root, host_dict, host_name, suite_resolutions) + + deps_elem = ET.SubElement(root, 'dependencies') + for dep in sorted(set(dependency_paths or [])): + d = ET.SubElement(deps_elem, 'dependency') + d.text = dep + + tree = ET.ElementTree(root) + ET.indent(tree, space=' ') + out_path = os.path.join(os.path.abspath(output_root), 'datatable.xml') + + # Serialise to a string buffer instead of writing directly, then route + # through write_if_changed so unchanged datatables don't get a fresh + # mtime (preserves CMake/Make's no-rebuild behaviour). + buf = io.StringIO() + tree.write(buf, encoding='unicode', xml_declaration=True) + content = buf.getvalue() + # ElementTree's text serialiser omits the trailing newline; add one + # for POSIX-text-file convention so the comparison is stable across + # editors that auto-append a newline. + if not content.endswith('\n'): + content += '\n' + write_if_changed(out_path, content, logger=logger) + + return out_path diff --git a/capgen/generator/group_cap.py b/capgen/generator/group_cap.py new file mode 100644 index 00000000..53927a7c --- /dev/null +++ b/capgen/generator/group_cap.py @@ -0,0 +1,1308 @@ +#!/usr/bin/env python3 + +"""Generate group-level cap Fortran source files. + +A group cap module (``ccpp___cap.F90``) contains one subroutine +per phase. Each subroutine: + +* USEs host-model modules for the variables it references. +* Declares control-variable dummy arguments. +* Declares transformation temporaries and optional pointer variables. +* Calls each scheme in the group in suite-XML order. +* Applies pre-call (forward) and post-call (backward) transformations. + +Subcycle loops wrap scheme calls enclosed by ```` elements in the +suite XML. + +Init deduplication (Section 12) is handled by the ``initialized`` guard array +declared in this module: an ``initialized(number_of_instances)`` integer array +is allocated at suite-init time and reset on suite-final. Each group's init +subroutine calls scheme ``_init`` routines only when ``initialized(inst) == 0`` +and then sets the element to 1. +""" + +import logging +import os +from typing import Dict, List, Optional, Set, Tuple + +from metadata.parse_tools import FORTRAN_CONDITIONAL_REGEX, open_if_changed +from metadata.variable_resolver import HostVarEntry +from generator.suite_types import _ptr_type_name_for_arg +from generator.suite_resolver import ( + ResolvedArg, + ResolvedCall, + ResolvedGroup, + ResolvedSubcycle, + _root_symbol, + iter_phase_calls, + iter_phase_subcycles, +) +from generator.trace import ( + emit_module_gate, + emit_trace_block, + ensure_error_unit_use, +) + +_INDENT = ' ' +_CONT = ' &' + +# Canonical phase order used for code-emission ordering in the group cap: +# subroutine layout, public declarations, scheme-import ``only:`` lists. +# Group caps do not contain a ``register`` phase (handled at suite-cap level); +# the order matches the user-facing ordering convention. +_GROUP_PHASE_ORDER = ('init', 'timestep_init', 'run', 'timestep_final', 'final') + +_CTRL_STDNAMES_ORDER = ( + 'suite_name', + 'group_name', + 'horizontal_loop_begin', + 'horizontal_loop_end', + 'thread_number', # paired-optional (with number_of_threads); ordered here when declared + 'number_of_threads', # paired-optional; ordered here when declared + 'number_of_physics_threads', + 'ccpp_error_code', + 'ccpp_error_message', + 'instance_number', +) + + +def _ctrl_entries_for_signature(host_dict, exclude=None): + """Return all control HostVarEntry objects in canonical order. + + Parameters + ---------- + host_dict : dict + Flat host+control variable dictionary. + exclude : set of str, optional + Standard names to exclude (e.g. ``{'suite_name'}`` at suite cap level). + """ + if host_dict is None: + return [] + exclude_set = set(exclude or []) + result = [] + seen: Set[str] = set() + for std_name in _CTRL_STDNAMES_ORDER: + if std_name in exclude_set: + continue + entry = host_dict.get(std_name) + if entry is not None and entry.is_control: + result.append(entry) + seen.add(std_name) + for std_name, entry in host_dict.items(): + if entry.is_control and std_name not in seen and std_name not in exclude_set: + result.append(entry) + return result + + +######################################################################## +# Fortran type declaration helpers +######################################################################## + +def _fortran_type_str(type_: str, kind: str) -> str: + """Return the Fortran type clause for a declaration. + + >>> _fortran_type_str('real', 'kind_phys') + 'real(kind=kind_phys)' + >>> _fortran_type_str('real', '') + 'real' + >>> _fortran_type_str('integer', '') + 'integer' + >>> _fortran_type_str('character', 'len=512') + 'character(len=512)' + >>> _fortran_type_str('logical', '') + 'logical' + >>> _fortran_type_str('gfs_statein_type', '') + 'type(gfs_statein_type)' + """ + t = type_.strip() + # DDT types (not intrinsic, not external:...) need type(...) syntax. + _INTRINSICS = frozenset({ + 'real', 'integer', 'character', 'logical', 'complex', 'double precision' + }) + if t.lower() not in _INTRINSICS and not t.lower().startswith('external:'): + if not t.lower().startswith('type('): + t = 'type({})'.format(t) + if kind: + if t.lower().startswith('character'): + return 'character({})'.format(kind) + return '{}(kind={})'.format(t, kind) + return t + + +def _dim_decl(dimensions: List[str]) -> str: + """Return the Fortran dimension attribute string for a declaration. + + Returns ``''`` for scalars, ``', dimension(d1, d2, ...)'`` for arrays. + The dimensions here use the declared dimension standard names as-is. + + >>> _dim_decl([]) + '' + >>> _dim_decl(['horizontal_dimension']) + ', dimension(horizontal_dimension)' + >>> _dim_decl(['horizontal_dimension', 'vertical_layer_dimension']) + ', dimension(horizontal_dimension, vertical_layer_dimension)' + """ + if not dimensions: + return '' + return ', dimension({})'.format(', '.join(dimensions)) + + +def _dim_decl_local(dimensions: List[str], host_dict) -> str: + """Return the Fortran dimension attribute using local names from host_dict. + + Each standard name in *dimensions* is resolved to the corresponding local + Fortran variable name. Falls back to the standard name when not found + (should not happen with valid metadata). + + Special case for ``horizontal_dimension``: the temp must match the + chunk slice the scheme actually receives at the call site + (``host_var(lb:ub, …)`` via :func:`_one_dim_part`). + Using the host's local name for ``horizontal_dimension`` (e.g. ``ncols``) + would over-size the temp and break the unit-conversion assignment + ``ps_l = factor * phys_state%ps(col_start:col_end)`` with a Fortran + shape mismatch. Emit ``dimension(:)`` instead, where ```` + and ```` are the host's local names for ``horizontal_loop_begin`` / + ``horizontal_loop_end``. + + >>> _dim_decl_local([], None) + '' + """ + if not dimensions: + return '' + locals_ = [] + for std_name in dimensions: + if std_name == 'horizontal_dimension': + lb_entry = host_dict.get('horizontal_loop_begin') if host_dict else None + ub_entry = host_dict.get('horizontal_loop_end') if host_dict else None + lb = lb_entry.local_name if lb_entry else 'horizontal_loop_begin' + ub = ub_entry.local_name if ub_entry else 'horizontal_loop_end' + locals_.append('{}:{}'.format(lb, ub)) + else: + entry = host_dict.get(std_name) if host_dict else None + locals_.append(entry.local_name if entry is not None else std_name) + return ', dimension({})'.format(', '.join(locals_)) + + +def _intent_clause(intent: str) -> str: + """Return the Fortran intent attribute. + + >>> _intent_clause('in') + ', intent(in)' + >>> _intent_clause('out') + ', intent(out)' + >>> _intent_clause('inout') + ', intent(inout)' + """ + return ', intent({})'.format(intent) + + +# Control variables that are output-only (set by the routine, not read). +_CTRL_OUT_STDNAMES = frozenset({'ccpp_error_code', 'ccpp_error_message'}) + + +def _ctrl_intent_for(standard_name: str) -> str: + """Return ``'out'`` for error-reporting control vars, else ``'in'``. + + >>> _ctrl_intent_for('ccpp_error_code') + 'out' + >>> _ctrl_intent_for('ccpp_error_message') + 'out' + >>> _ctrl_intent_for('horizontal_loop_begin') + 'in' + """ + return 'out' if standard_name in _CTRL_OUT_STDNAMES else 'in' + + +def _ctrl_local(host_dict, standard_name: str): + """Return the local Fortran name for a control standard_name, or ``None``.""" + if not host_dict: + return None + entry = host_dict.get(standard_name) + return entry.local_name if entry is not None else None + + +######################################################################## +# USE-statement collection +######################################################################## + +def _active_std_names(active: str) -> Set[str]: + """Return the set of identifier tokens from an active expression string. + + Uses the same tokeniser as ``_translate_active_expr`` so that only word + tokens (potential standard names) are returned, not operators or literals. + + >>> sorted(_active_std_names('my_flag .eqv. .true.')) + ['my_flag'] + >>> sorted(_active_std_names('a .or. b')) + ['a', 'b'] + >>> _active_std_names('') + set() + """ + if not active: + return set() + result: Set[str] = set() + for m in FORTRAN_CONDITIONAL_REGEX.finditer(active): + tok = m.group(0) + # Skip Fortran keywords / literals / operators captured by the regex. + if tok.strip() and tok[0].isalpha() and '_' not in tok[:1]: + # Only word-like tokens could be standard names; filter out + # Fortran logical literals and operators (.true., .false., .not., ...) + result.add(tok) + elif tok[0] == '_' or (tok[0].isalpha() and tok.isidentifier()): + result.add(tok) + return result + + +def _collect_group_uses( + resolved_group: ResolvedGroup, + host_dict, +) -> Dict[str, Set[str]]: + """Collect ``{module: {symbol, ...}}`` across all phases of a group. + + Includes direct argument symbols, dimension helper symbols, and variables + referenced in ``active`` conditional expressions. + + Parameters + ---------- + resolved_group : ResolvedGroup + host_dict : dict + Flat host+control variable dictionary (for dimension look-ups). + + Returns + ------- + dict + """ + uses: Dict[str, Set[str]] = {} + + def _add(mod: Optional[str], sym: str) -> None: + if mod is not None: + uses.setdefault(mod, set()).add(sym) + + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: + # Direct argument symbol. + if arg.source != 'control': + _add(arg.module_name, arg.root_symbol) + # Dimension helper symbols (non-control only). + # USE the access-path root, not entry.local_name — DDT-component + # entries have local_name = component (not a free module symbol). + for dim_std in arg.used_dim_std_names: + entry = host_dict.get(dim_std) + if entry is not None and entry.module_name is not None: + _add(entry.module_name, _root_symbol(entry.access_path)) + # Variables referenced in active expressions (Gap 1). + # Same access-path-root rule applies (DDT-component flags + # reach the cap via the DDT instance, not the component). + for std_name in _active_std_names(arg.active): + entry = host_dict.get(std_name) if host_dict else None + if entry is not None and entry.module_name is not None: + _add(entry.module_name, _root_symbol(entry.access_path)) + # Constituent extra symbols (index_of_X, etc.) live in the + # suite cap module along with the constituent arrays. + if arg.source == 'constituent' and arg.constituent_module_name: + for sym in arg.constituent_extra_symbols: + _add(arg.constituent_module_name, sym) + + # Also add dim_uses already collected during resolution. + for mod, syms in resolved_group.dim_uses.items(): + uses.setdefault(mod, set()).update(syms) + + return uses + + +def _collect_kinds_used(resolved_group: ResolvedGroup) -> List[str]: + """Collect kind parameter *names* referenced by transformation temporaries. + + Mirrors the kind-resolution logic in :func:`_generate_phase_subroutine` + (the only place a group cap emits ``kind=`` directly). Returned + names are sorted alphabetically. Excluded: + + * character ``len=...`` specifiers — not kind parameters. + * bare integer literals (``kind = 8``, ``kind = 4``, ...) — valid + Fortran kind specifiers but not module symbols, so they must not + appear in ``use ccpp_kinds, only: ...``. They flow through to the + temp declaration (``real(kind=8)``) and to numeric-literal suffixes + (``1.0_8``) unchanged; only the USE list needs to filter them out. + """ + kinds: Set[str] = set() + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: + if not arg.temp_name: + continue + kind = arg.kind_scheme or ( + arg.host_entry.kind if arg.host_entry else '' + ) + if not kind: + continue + if kind.startswith('len='): + continue + if kind.isdigit(): + continue + kinds.add(kind) + return sorted(kinds) + + +######################################################################## +# Control variable dummy argument handling +######################################################################## + +def _collect_control_args(resolved_group: ResolvedGroup) -> List[ResolvedArg]: + """Return deduplicated control-variable arguments for the group subroutine. + + Returns one ResolvedArg per unique standard_name, in a consistent order. + """ + seen: Dict[str, ResolvedArg] = {} + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: + if arg.source == 'control' and arg.standard_name not in seen: + seen[arg.standard_name] = arg + return list(seen.values()) + + +def _extra_dim_ctrl_entries( + phase_items, + phase: str, + ctrl_args: List[ResolvedArg], + host_dict, +) -> List[HostVarEntry]: + """Return HostVarEntry objects for control vars needed but not in ctrl_args. + + Covers two cases (Gap 3): + + 1. ``instance_number`` — needed for state-array indexing in init/final/ + timestep_init/timestep_final, and for suite-data access + ``ccpp_suite_data(inst_num)%...`` in any phase that references suite vars. + 2. Any control var appearing only in dimension subscripts (``used_dim_std_names``). + """ + if host_dict is None: + return [] + already = {a.standard_name for a in ctrl_args if a.host_entry is not None} + extras: Dict[str, HostVarEntry] = {} + + has_suite_vars = any( + arg.source == 'suite' + for resolved_call in iter_phase_calls(phase_items) + for arg in resolved_call.args + ) + needs_inst = ( + phase in ('init', 'final', 'timestep_init', 'timestep_final') + or has_suite_vars + ) + if needs_inst: + inst_entry = host_dict.get('instance_number') + if inst_entry is not None and 'instance_number' not in already: + extras['instance_number'] = inst_entry + + for resolved_call in iter_phase_calls(phase_items): + for arg in resolved_call.args: + for dim_std in arg.used_dim_std_names: + if dim_std in already or dim_std in extras: + continue + entry = host_dict.get(dim_std) + if entry is not None and entry.is_control: + extras[dim_std] = entry + + # Subcycle loop bounds resolved from a CCPP standard name in the suite + # XML need the same dummy-arg threading when the resolved entry is a + # control variable (host-module entries are USE'd instead and are + # handled by ``_collect_dim_uses``). Walk all nesting levels — + # each level's bound is independently a candidate dummy arg. + for item in iter_phase_subcycles(phase_items): + std = item.loop_std_name + if not std or std in already or std in extras: + continue + entry = host_dict.get(std) + if entry is not None and entry.is_control: + extras[std] = entry + + return list(extras.values()) + + +######################################################################## +# Fortran source line helpers +######################################################################## + +def _use_statements(uses: Dict[str, Set[str]]) -> List[str]: + """Generate sorted USE statements. + + >>> lines = _use_statements({'mod_a': {'sym1', 'sym2'}, 'mod_b': {'sym3'}}) + >>> lines[0] + ' use mod_a, only: sym1, sym2' + >>> lines[1] + ' use mod_b, only: sym3' + """ + result = [] + for mod in sorted(uses): + syms = ', '.join(sorted(uses[mod])) + result.append('{}use {}, only: {}'.format(_INDENT, mod, syms)) + return result + + +def _collect_scheme_uses(resolved_group: ResolvedGroup) -> List[Tuple[str, str, List[str]]]: + """Return ``[(scheme_name, module_name, [phase_routine, ...]), ...]``. + + Schemes are listed in first-seen order across phases (in canonical phase + iteration order); within each scheme the phase routines are listed in + canonical phase order. Each phase routine is the Fortran subroutine name + ``_``. + + ``module_name`` is the Fortran module that exports those subroutines — + typically equal to the scheme name, but overridden by an explicit + ``module_name`` attribute in the scheme's ``[ccpp-table-properties]``. + """ + seen_schemes: Dict[str, Set[str]] = {} + scheme_modules: Dict[str, str] = {} + order: List[str] = [] + for phase in _GROUP_PHASE_ORDER: + for resolved_call in iter_phase_calls(resolved_group.phase_calls.get(phase, [])): + if resolved_call.scheme_name not in seen_schemes: + seen_schemes[resolved_call.scheme_name] = set() + order.append(resolved_call.scheme_name) + seen_schemes[resolved_call.scheme_name].add(phase) + # resolved_call.scheme_module is empty for old/legacy ResolvedCall + # objects built in tests; fall back to the scheme name so emission + # still works. + scheme_modules[resolved_call.scheme_name] = ( + resolved_call.scheme_module or resolved_call.scheme_name + ) + result: List[Tuple[str, str, List[str]]] = [] + for sname in order: + phases_present = [p for p in _GROUP_PHASE_ORDER if p in seen_schemes[sname]] + syms = ['{}_{}'.format(sname, p) for p in phases_present] + result.append((sname, scheme_modules[sname], syms)) + return result + + +def _scheme_use_statements(resolved_group: ResolvedGroup) -> List[str]: + """Generate ``use , only: _, ...`` lines. + + Schemes are emitted in first-seen order; phase routines within each + ``only:`` clause follow the canonical phase order. When the scheme's + ``[ccpp-table-properties]`` declares a ``module_name`` distinct from + the scheme name, the USE statement targets that module. + """ + return [ + '{}use {}, only: {}'.format(_INDENT, mod, ', '.join(syms)) + for _sname, mod, syms in _collect_scheme_uses(resolved_group) + if syms + ] + + +######################################################################## +# Pre/post-call transformation code +######################################################################## + +def _transform_comment(arg: ResolvedArg, reverse: bool = False) -> str: + """Compose the trailing inline comment for a transform assignment. + + Lists every active transform (unit conversion, kind change, vertical + flip) so a reader can tell at a glance what the generated copy does. + + Identity unit conversions (registered for dimensionally-equivalent + spellings such as ``J kg-1`` ↔ ``m2 s-2``, where the formula is just + ``{var}``) are suppressed: the assignment carries no scaling factor + and labelling it as a "unit conversion" is misleading. Detection is + done by comparing the rendered transform expression against the raw + operand — equality means the formula returned the variable unchanged. + """ + bits: List[str] = [] + if arg.needs_unit_transform or arg.needs_kind_transform: + # Suppress identity conversions (formula '{var}' for equivalent + # units; kinds also matching). + if reverse: + is_identity = (arg.unit_backward == arg.temp_name) + else: + is_identity = (arg.unit_forward == arg.call_expr) + if not is_identity: + if reverse: + bits.append('unit conversion: {} to {}'.format( + arg.kind_scheme or '', arg.kind_host or '', + )) + else: + bits.append('unit conversion: {} to {}'.format( + arg.kind_host or '', arg.kind_scheme or '', + )) + if arg.needs_vert_flip: + bits.append('vertical flip (top_at_one mismatch)') + if not bits: + return '' + return '! ' + '; '.join(bits) + + +def _active_required_guard_lines( + arg: ResolvedArg, + scheme_name: str, + phase: str, + errflg_local: Optional[str], + errmsg_local: Optional[str], + indent: str, +) -> List[str]: + """Runtime guard for a non-optional scheme arg whose host declares ``active``. + + The host says the variable is only valid when ``active`` is true. + The scheme demands the variable unconditionally. We emit a runtime + check at the call site so a violation surfaces as a clean errflg/errmsg + rather than as a silent read of unallocated/stale memory. + + Emitted before any transform pre-emission and before the call itself. + If the host did not declare both ``ccpp_error_code`` and + ``ccpp_error_message`` (extremely unusual), the guard is omitted — + there is no way to report the violation. + """ + if not arg.active_local or arg.is_optional: + return [] + if not errflg_local or not errmsg_local: + return [] + msg = ( + "scheme '{scheme}' phase '{phase}' requires variable " + "'{std}' but host active condition ({active}) is false".format( + scheme=scheme_name, phase=phase, + std=arg.standard_name, active=arg.active, + ) + ) + return [ + '{}if (.not. ({})) then'.format(indent, arg.active_local), + "{} {} = \"{}\"".format(indent, errmsg_local, msg), + '{} {} = 1'.format(indent, errflg_local), + '{} return'.format(indent), + '{}end if'.format(indent), + ] + + +def _pre_call_lines(arg: ResolvedArg) -> List[str]: + """Generate pre-call Fortran lines for one argument.""" + lines = [] + if arg.transform_case == 1: + return lines + + indent = _INDENT * 2 + + if arg.transform_case == 2: + # Optional, no transform: pointer assignment. + lines.append('{}if ({}) then'.format(indent, arg.active_local or '.true.')) + lines.append('{} {}%ptr => {}'.format(indent, arg.ptr_name, arg.call_expr)) + lines.append('{}else'.format(indent)) + lines.append('{} nullify({}%ptr)'.format(indent, arg.ptr_name)) + lines.append('{}end if'.format(indent)) + + elif arg.transform_case == 3: + # Transform, not optional. + if arg.unit_forward: + comment = _transform_comment(arg) + sep = ' ' if comment else '' + lines.append('{}{} = {}{}{}'.format( + indent, arg.temp_name, arg.unit_forward, sep, comment + )) + + elif arg.transform_case == 4: + # Transform + optional pointer. + lines.append('{}if ({}) then'.format(indent, arg.active_local or '.true.')) + if arg.unit_forward: + lines.append('{} {} = {}'.format(indent, arg.temp_name, arg.unit_forward)) + lines.append('{} {}%ptr => {}'.format(indent, arg.ptr_name, arg.temp_name)) + lines.append('{}else'.format(indent)) + lines.append('{} nullify({}%ptr)'.format(indent, arg.ptr_name)) + lines.append('{}end if'.format(indent)) + + return lines + + +def _post_call_lines(arg: ResolvedArg) -> List[str]: + """Generate post-call Fortran lines for one argument.""" + lines = [] + if arg.transform_case == 1: + return lines + + indent = _INDENT * 2 + + if arg.transform_case == 2: + lines.append('{}nullify({}%ptr)'.format(indent, arg.ptr_name)) + + elif arg.transform_case == 3: + if arg.unit_backward: + comment = _transform_comment(arg, reverse=True) + sep = ' ' if comment else '' + lines.append('{}{} = {}{}{}'.format( + indent, arg.call_expr, arg.unit_backward, sep, comment + )) + + elif arg.transform_case == 4: + lines.append('{}if ({}) then'.format(indent, arg.active_local or '.true.')) + lines.append('{} nullify({}%ptr)'.format(indent, arg.ptr_name)) + if arg.unit_backward: + lines.append('{} {} = {}'.format(indent, arg.call_expr, arg.unit_backward)) + lines.append('{}end if'.format(indent)) + + return lines + + +def _call_arg_expr(arg: ResolvedArg) -> str: + """Return the Fortran expression to pass for this argument at the call site.""" + if arg.transform_case == 1: + return arg.call_expr + elif arg.transform_case == 2: + return '{}%ptr'.format(arg.ptr_name) + elif arg.transform_case == 3: + return arg.temp_name + else: # 4 + return '{}%ptr'.format(arg.ptr_name) + + +######################################################################## +# Scheme-call code generation helper +######################################################################## + +def _max_subcycle_depth(items) -> int: + """Return the maximum subcycle nesting depth in *items*. + + A flat list of scheme calls has depth 0; a single ```` + wrapping schemes has depth 1; a subcycle wrapping a subcycle has + depth 2; and so on. Used to pre-declare one integer loop counter + per nesting level (``ccpp_loop_counter``, ``ccpp_loop_counter_2``, + ``ccpp_loop_counter_3``, ...). + """ + depth = 0 + for item in items: + if isinstance(item, ResolvedSubcycle): + depth = max(depth, 1 + _max_subcycle_depth(item.calls)) + return depth + + +def _loop_counter_name(depth: int) -> str: + """Return the loop-counter Fortran identifier for *depth* (1-based). + + Depth 1 (outermost / single level) returns ``'ccpp_loop_counter'`` + so existing single-subcycle tests and host expectations are + unchanged. Deeper levels get ``ccpp_loop_counter_``. + """ + if depth <= 1: + return 'ccpp_loop_counter' + return 'ccpp_loop_counter_{}'.format(depth) + + +def _emit_phase_items( + items, indent: str, lines: List[str], depth: int, + phase: str = '', + errflg_local: Optional[str] = None, + errmsg_local: Optional[str] = None, +) -> None: + """Recursively emit Fortran for a list of :data:`PhaseItem` objects. + + A :class:`ResolvedCall` becomes a single scheme call (with pre/post + transforms). A :class:`ResolvedSubcycle` becomes a ``do`` loop + that wraps recursively-emitted children, with one fresh integer + loop variable per nesting level. + """ + for item in items: + if isinstance(item, ResolvedCall): + _emit_one_call( + item, indent, lines, phase, errflg_local, errmsg_local, + ) + elif isinstance(item, ResolvedSubcycle): + counter = _loop_counter_name(depth) + lines.append( + '{}do {} = 1, {}'.format(indent, counter, item.loop) + ) + _emit_phase_items( + item.calls, indent + _INDENT, lines, depth=depth + 1, + phase=phase, + errflg_local=errflg_local, + errmsg_local=errmsg_local, + ) + lines.append('{}end do'.format(indent)) + lines.append('') + + +def _emit_one_call( + resolved_call: ResolvedCall, + indent: str, + lines: List[str], + phase: str = '', + errflg_local: Optional[str] = None, + errmsg_local: Optional[str] = None, +) -> None: + """Append Fortran lines for a single scheme call (with transforms + errcheck).""" + # Pre-call: runtime guard for any non-optional arg whose host declares + # ``active = (...)``. Emitted before transforms so an inactive-but-required + # var bails out with a clear error rather than reading host memory through + # the transform pipeline. + for arg in resolved_call.args: + lines.extend(_active_required_guard_lines( + arg, resolved_call.scheme_name, resolved_call.phase, + errflg_local, errmsg_local, indent, + )) + + # Pre-call transformations. + for arg in resolved_call.args: + lines.extend(_pre_call_lines(arg)) + + call_args_exprs = [ + '{}={}'.format(a.scheme_local_name, _call_arg_expr(a)) + for a in resolved_call.args + ] + call_name = '{}_{}'.format(resolved_call.scheme_name, resolved_call.phase) + + if call_args_exprs: + lines.append('{}call {}( &'.format(indent, call_name)) + for i, expr in enumerate(call_args_exprs): + sep = ', &' if i < len(call_args_exprs) - 1 else ')' + lines.append('{} {}{}'.format(indent, expr, sep)) + else: + lines.append('{}call {}()'.format(indent, call_name)) + + errflg_arg = next( + (a for a in resolved_call.args if a.standard_name == 'ccpp_error_code'), None + ) + if errflg_arg is not None: + lines.append('{}if ({} /= 0) return'.format(indent, _call_arg_expr(errflg_arg))) + + for arg in resolved_call.args: + lines.extend(_post_call_lines(arg)) + lines.append('') + + +######################################################################## +# State machine guards +######################################################################## + +def _state_entry_guard( + phase: str, + inst_idx: str, + errflg_local: Optional[str], + errmsg_local: Optional[str], + sub_label: str, + indent: str, +) -> List[str]: + """Return Fortran lines that validate the group state on phase entry. + + Per the design table: + + ===================== ============================================ + Phase Required state + ===================== ============================================ + ``init`` ``UNINITIALIZED`` (idempotent skip if ``INITIALIZED``) + ``timestep_init`` ``== INITIALIZED`` + ``run`` ``== IN_TIMESTEP`` + ``timestep_final`` ``== IN_TIMESTEP`` + ``final`` ``>= INITIALIZED`` (idempotent skip if ``UNINITIALIZED``) + ===================== ============================================ + + Invalid state sets ``errflg = 1``, populates ``errmsg``, and returns. + + If the host did not declare ``ccpp_error_code`` / ``ccpp_error_message`` + (extremely unusual), the guard is omitted — there is no way to report the + error and the routine cannot proceed without somewhere to write it. + """ + if not errflg_local or not errmsg_local: + return [] + + state_var = 'ccpp_group_state({})'.format(inst_idx) + msg = "ccpp_{}: invalid group state".format(sub_label) + + def _err_block(condition: str) -> List[str]: + return [ + '{}if ({}) then'.format(indent, condition), + "{} {} = '{}'".format(indent, errmsg_local, msg), + '{} {} = 1'.format(indent, errflg_local), + '{} return'.format(indent), + '{}end if'.format(indent), + '', + ] + + if phase == 'init': + # Idempotent skip when already INITIALIZED; error if past INITIALIZED + # (e.g. IN_TIMESTEP) — init must come from UNINITIALIZED. + lines: List[str] = [ + '{}if ({} == CCPP_GROUP_INITIALIZED) return'.format(indent, state_var), + '', + ] + lines.extend(_err_block( + '{} /= CCPP_GROUP_UNINITIALIZED'.format(state_var) + )) + return lines + if phase == 'timestep_init': + return _err_block('{} /= CCPP_GROUP_INITIALIZED'.format(state_var)) + if phase == 'run': + return _err_block('{} /= CCPP_GROUP_IN_TIMESTEP'.format(state_var)) + if phase == 'timestep_final': + return _err_block('{} /= CCPP_GROUP_IN_TIMESTEP'.format(state_var)) + if phase == 'final': + # Idempotent skip when already UNINITIALIZED; INITIALIZED and + # IN_TIMESTEP are both valid entry states (UNINITIALIZED is the only + # state value < INITIALIZED, so no error block is reachable here). + return [ + '{}if ({} == CCPP_GROUP_UNINITIALIZED) return'.format(indent, state_var), + '', + ] + return [] + + +######################################################################## +# Subroutine generator +######################################################################## + +def _generate_phase_subroutine( + suite_name: str, + group_name: str, + phase: str, + phase_items, + ctrl_entries, + host_dict, +) -> List[str]: + """Generate one phase subroutine for a group cap. + + For the ``init`` phase the body is wrapped in a state guard + (``ccpp_group_state(1) < CCPP_GROUP_INITIALIZED``) and the state is set + to ``CCPP_GROUP_INITIALIZED`` at the end. + + For the ``final`` phase the state is reset to ``CCPP_GROUP_UNINITIALIZED`` + at the end. + + ``ResolvedSubcycle`` items in *phase_items* generate ``do`` loops (run + phase only). + + Returns a list of Fortran source lines (no trailing newlines). + """ + # Short Fortran symbol; module name already namespaces ``_`` + # as ``ccpp___cap_mp__`` at link time, keeping + # the mangled global name under Intel's ~90-char threshold. The long + # form ``__`` is kept in ``sub_label`` below for + # trace strings and error messages (string literals have no length cap). + sub_name = '{}_{}'.format(group_name, phase) + lines: List[str] = [] + + # ---- subroutine declaration ------------------------------------------ + ctrl_local_names = [e.local_name for e in ctrl_entries] + + if ctrl_local_names: + lines.append('{}subroutine {}( &'.format(_INDENT, sub_name)) + for i, lname in enumerate(ctrl_local_names): + sep = ', &' if i < len(ctrl_local_names) - 1 else ')' + lines.append('{} {}{}'.format(_INDENT, lname, sep)) + else: + lines.append('{}subroutine {}()'.format(_INDENT, sub_name)) + + # ---- dummy argument declarations ------------------------------------ + if ctrl_local_names: + lines.append('') + for entry in ctrl_entries: + # Character control dummies always use len=* so the host's specific + # length doesn't propagate into the generated signature. + kind = 'len=*' if entry.type.strip().lower() == 'character' else entry.kind + t = _fortran_type_str(entry.type, kind) + intent = _intent_clause(_ctrl_intent_for(entry.standard_name)) + dim = _dim_decl(entry.dimensions) + lines.append( + '{}{}{}{} :: {}'.format(_INDENT * 2, t, intent, dim, entry.local_name) + ) + + # ---- local variable declarations (transformation temps, subcycle counter) + local_decls: List[str] = [] + # Each subcycle nesting level needs its own integer loop variable. + # The outermost (and only one, in the single-level case) is named + # ``ccpp_loop_counter`` to preserve the existing single-level + # convention; deeper levels are ``ccpp_loop_counter_2``, + # ``ccpp_loop_counter_3``, ... so nested loops have distinct vars. + max_depth = _max_subcycle_depth(phase_items) + for d in range(1, max_depth + 1): + name = 'ccpp_loop_counter' if d == 1 else 'ccpp_loop_counter_{}'.format(d) + local_decls.append('{}integer :: {}'.format(_INDENT * 2, name)) + + seen_temp_names: Set[str] = set() + seen_ptr_names: Set[str] = set() + for resolved_call in iter_phase_calls(phase_items): + for arg in resolved_call.args: + if arg.temp_name and arg.temp_name not in seen_temp_names: + seen_temp_names.add(arg.temp_name) + t = _fortran_type_str( + arg.host_entry.type if arg.host_entry else arg.suite_var.type_, + arg.kind_scheme or (arg.host_entry.kind if arg.host_entry else ''), + ) + # Use scheme dimensions (local names) for the temp declaration + # so the temp is sized for the chunk the scheme actually receives + # (Gap 2: avoids emitting standard names in the declaration). + dim = _dim_decl_local(arg.scheme_dimensions, host_dict) + # Transform-case 4 emits ``%ptr => ``, so the temp + # must have the TARGET attribute or pointer assignment is illegal. + target_attr = ', target' if arg.ptr_name else '' + local_decls.append( + '{}{}{}{} :: {}'.format( + _INDENT * 2, t, dim, target_attr, arg.temp_name + ) + ) + if arg.ptr_name and arg.ptr_name not in seen_ptr_names: + seen_ptr_names.add(arg.ptr_name) + ptr_tname = _ptr_type_name_for_arg(arg, resolved_call.scheme_name) + local_decls.append( + '{}type({}) :: {}'.format(_INDENT * 2, ptr_tname, arg.ptr_name) + ) + + if local_decls: + lines.append('') + lines.extend(local_decls) + + lines.append('') + + call_indent = _INDENT * 2 + + inst_idx = _instance_idx(host_dict) + errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') + errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') + # Long form used as the trace-message label and in runtime error + # strings so grep against logs still finds the suite + group context. + sub_label = '{}_{}_{}'.format(suite_name, group_name, phase) + + # ---- trace block (always emitted; gated by the module ``trace`` + # parameter so the I/O is dead-code-eliminated when trace=.false.). + # Placed before errmsg/errflg init so the write references no + # intent(out) dummy and fires even when a state guard then bails. + trace_lines = emit_trace_block(sub_label, ctrl_entries, call_indent) + if trace_lines: + lines.extend(trace_lines) + lines.append('') + + # ---- initialize error reporting vars ------------------------------- + if errflg_local and errmsg_local: + lines.append("{}{} = ''".format(call_indent, errmsg_local)) + lines.append('{}{} = 0'.format(call_indent, errflg_local)) + lines.append('') + + # ---- phase entry state guards -------------------------------------- + lines.extend( + _state_entry_guard( + phase, inst_idx, errflg_local, errmsg_local, sub_label, call_indent + ) + ) + + # ---- scheme calls --------------------------------------------------- + _emit_phase_items( + phase_items, call_indent, lines, depth=1, + phase=phase, + errflg_local=errflg_local, + errmsg_local=errmsg_local, + ) + + # ---- post-call state transitions ------------------------------------ + if phase == 'init': + lines.append( + '{}ccpp_group_state({}) = CCPP_GROUP_INITIALIZED'.format( + call_indent, inst_idx + ) + ) + lines.append('') + elif phase == 'timestep_init': + lines.append( + '{}ccpp_group_state({}) = CCPP_GROUP_IN_TIMESTEP'.format( + call_indent, inst_idx + ) + ) + lines.append('') + elif phase == 'timestep_final': + lines.append( + '{}ccpp_group_state({}) = CCPP_GROUP_INITIALIZED'.format( + call_indent, inst_idx + ) + ) + lines.append('') + elif phase == 'final': + lines.append( + '{}ccpp_group_state({}) = CCPP_GROUP_UNINITIALIZED'.format( + call_indent, inst_idx + ) + ) + lines.append('') + + lines.append('{}end subroutine {}'.format(_INDENT, sub_name)) + return lines + + +######################################################################## +# Group cap module generator +######################################################################## + +def _instance_idx(host_dict) -> str: + """Return the Fortran index expression for the current model instance. + + Returns the local name of the ``instance_number`` control variable when the + host declares it; otherwise returns the literal ``'1'`` for single-instance + models. + + >>> class _FakeEntry: + ... local_name = 'inst_num' + >>> class _FakeDict(dict): + ... pass + >>> d = _FakeDict({'instance_number': _FakeEntry()}) + >>> _instance_idx(d) + 'inst_num' + >>> _instance_idx({}) + '1' + """ + entry = host_dict.get('instance_number') if host_dict else None + return entry.local_name if entry is not None else '1' + + +def _instance_local(host_dict) -> Optional[str]: + """Return the host's local name for ``instance_number``, or ``None``. + + Companion to :func:`_instance_idx`. Callers use the ``None`` return + to decide whether to inject ``instance_number`` into a subroutine's + signature (i.e. whether the host opted into the multi-instance API). + + >>> class _FakeEntry: + ... local_name = 'inst_num' + >>> _instance_local({'instance_number': _FakeEntry()}) + 'inst_num' + >>> _instance_local({}) is None + True + """ + entry = host_dict.get('instance_number') if host_dict else None + return entry.local_name if entry is not None else None + + +def _generate_state_alloc(suite_name: str, group_name: str) -> List[str]: + """Generate the ``ccpp___state_alloc`` subroutine. + + The subroutine always accepts ``number_of_instances`` as an explicit + ``intent(in)`` integer argument so the caller (the suite cap's init + routine) can supply the count at runtime without the group cap needing to + USE any host module. + + Idempotent: a second call after the array is already allocated is a + no-op. The suite cap calls this once per ``_init`` invocation, + so when multiple instances initialize, only the first allocates and + initialises the state array; subsequent calls return immediately to + avoid clobbering peer-instance state slots. Matches the + ``_suite_state_alloc`` pattern. + """ + # Short Fortran symbol; the module ``ccpp___cap`` + # already namespaces this routine at link time. + sub_name = '{}_state_alloc'.format(group_name) + i1 = _INDENT + i2 = _INDENT * 2 + lines = [ + '', + '{}subroutine {}(number_of_instances, errmsg, errflg)'.format(i1, sub_name), + '', + '{}integer, intent(in) :: number_of_instances'.format(i2), + '{}character(len=*), intent(out) :: errmsg'.format(i2), + '{}integer, intent(out) :: errflg'.format(i2), + '', + "{}errmsg = ''".format(i2), + '{}errflg = 0'.format(i2), + '{}if (allocated(ccpp_group_state)) return'.format(i2), + '{}allocate(ccpp_group_state(number_of_instances))'.format(i2), + '{}ccpp_group_state(:) = CCPP_GROUP_UNINITIALIZED'.format(i2), + '', + '{}end subroutine {}'.format(i1, sub_name), + ] + return lines + + +def _generate_state_dealloc(suite_name: str, group_name: str) -> List[str]: + """Generate the ``_state_dealloc`` subroutine (Fortran symbol). + + The module name ``ccpp___cap`` already namespaces this + routine, so the short Fortran name keeps the mangled global symbol + under Intel's ~90-char limit. + """ + sub_name = '{}_state_dealloc'.format(group_name) + i1 = _INDENT + i2 = _INDENT * 2 + return [ + '', + '{}subroutine {}(errmsg, errflg)'.format(i1, sub_name), + '', + '{}character(len=*), intent(out) :: errmsg'.format(i2), + '{}integer, intent(out) :: errflg'.format(i2), + '', + "{}errmsg = ''".format(i2), + '{}errflg = 0'.format(i2), + '{}if (allocated(ccpp_group_state)) deallocate(ccpp_group_state)'.format(i2), + '', + '{}end subroutine {}'.format(i1, sub_name), + ] + + +def _generate_group_cap( + suite_name: str, + group_name: str, + resolved_group: ResolvedGroup, + host_dict, + trace: bool = False, +) -> List[str]: + """Generate the full group cap module source lines. + + Parameters + ---------- + suite_name : str + group_name : str + resolved_group : ResolvedGroup + host_dict : dict + Flat host+control dictionary. + + Returns + ------- + list of str (without trailing newlines) + """ + mod_name = 'ccpp_{}_{}_{}'.format(suite_name, group_name, 'cap') + # Short Fortran symbols for the state-management subroutines; the + # module name carries ``ccpp___`` and keeps the mangled + # global symbol (``_mp_``) under Intel's ~90-char limit. + alloc_sub = '{}_state_alloc'.format(group_name) + dealloc_sub = '{}_state_dealloc'.format(group_name) + lines: List[str] = [] + + # ---- module header -------------------------------------------------- + lines.append( + '! ccpp_{}_{}_cap.F90 -- generated by ccpp_capgen, do not edit'.format( + suite_name, group_name + ) + ) + lines.append('module {}'.format(mod_name)) + lines.append('') + + # ---- USE statements ------------------------------------------------- + uses = _collect_group_uses(resolved_group, host_dict) + + # Add USE for types module when optional pointer args are present. + # Build the wrapper name via the per-arg helper so any + # unsupported shape (e.g. character(len=*)) raises a CCPPError + # naming the offending scheme + argument. + ptr_type_names: Set[str] = set() + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: + if arg.ptr_name: + ptr_type_names.add( + _ptr_type_name_for_arg(arg, resolved_call.scheme_name) + ) + if ptr_type_names: + types_mod = 'ccpp_{}_types'.format(suite_name) + uses[types_mod] = ptr_type_names + + # USE ccpp_kinds for any kind parameter referenced in transformation + # temporaries declared in this group (e.g. ``real(kind=kind_phys)``). + kind_names = _collect_kinds_used(resolved_group) + if kind_names: + uses['ccpp_kinds'] = set(kind_names) + + use_lines = _use_statements(uses) + use_lines.extend(_scheme_use_statements(resolved_group)) + # Trace block writes to error_unit; ensure the USE is present. + ensure_error_unit_use(use_lines, _INDENT) + lines.extend(use_lines) + if use_lines: + lines.append('') + + lines.append('{}implicit none'.format(_INDENT)) + lines.append('{}private'.format(_INDENT)) + + # Public physics subroutines, in canonical phase order. + # Always emit all phases so the suite cap can rely on the state machine + # transitioning through every phase, even when a group has no scheme + # routine for a particular phase. + for phase in _GROUP_PHASE_ORDER: + sub_name = '{}_{}'.format(group_name, phase) + lines.append('{}public :: {}'.format(_INDENT, sub_name)) + + # Public state management subroutines. + lines.append('{}public :: {}'.format(_INDENT, alloc_sub)) + lines.append('{}public :: {}'.format(_INDENT, dealloc_sub)) + + # ---- state machine module-level declarations ------------------------- + lines.append('') + lines.append('{}integer, private, parameter :: CCPP_GROUP_UNINITIALIZED = 0'.format(_INDENT)) + lines.append('{}integer, private, parameter :: CCPP_GROUP_INITIALIZED = 1'.format(_INDENT)) + lines.append('{}integer, private, parameter :: CCPP_GROUP_IN_TIMESTEP = 2'.format(_INDENT)) + lines.append('{}integer, private, allocatable :: ccpp_group_state(:)'.format(_INDENT)) + + # ---- trace gate ------------------------------------------------------- + # Module-level compile-time toggle; flip to .true. (or pass --trace at + # generation time) to enable the per-subroutine trace writes. When + # .false., the gated writes are dead-code-eliminated by the compiler + # but the control dummies remain syntactically referenced, which + # silences strict unused-dummy warnings (Intel oneAPI in particular). + lines.extend(emit_module_gate(trace, _INDENT)) + + lines.append('') + lines.append('contains') + lines.append('') + + # ---- subroutines per phase ------------------------------------------ + # All phases share the same uniform control arg signature (excluding + # suite_name and group_name which are consumed at higher dispatch levels). + ctrl_sig_entries = _ctrl_entries_for_signature( + host_dict, exclude={'suite_name', 'group_name'} + ) + for phase in _GROUP_PHASE_ORDER: + phase_items = resolved_group.phase_calls.get(phase, []) + sub_lines = _generate_phase_subroutine( + suite_name, group_name, phase, phase_items, ctrl_sig_entries, host_dict + ) + lines.extend(sub_lines) + lines.append('') + + # ---- state management subroutines ----------------------------------- + lines.extend(_generate_state_alloc(suite_name, group_name)) + lines.append('') + lines.extend(_generate_state_dealloc(suite_name, group_name)) + lines.append('') + + lines.append('end module {}'.format(mod_name)) + return lines + + +def _ctrl_args_for_phase(resolved_group: ResolvedGroup, phase: str) -> List[ResolvedArg]: + """Return control args used in a specific phase, deduplicated.""" + seen: Dict[str, ResolvedArg] = {} + for resolved_call in iter_phase_calls(resolved_group.phase_calls.get(phase, [])): + for arg in resolved_call.args: + if arg.source == 'control' and arg.standard_name not in seen: + seen[arg.standard_name] = arg + return list(seen.values()) + + +######################################################################## +# Public API +######################################################################## + +def write_group_cap( + suite_name: str, + group_name: str, + resolved_group: ResolvedGroup, + host_dict, + output_root: str, + logger: Optional[logging.Logger] = None, + trace: bool = False, +) -> str: + """Write the group cap Fortran module to *output_root*. + + Parameters + ---------- + suite_name : str + group_name : str + resolved_group : ResolvedGroup + Resolved call information for this group. + host_dict : dict + Flat host+control variable dictionary. + output_root : str + Output directory (created if absent). + + Returns + ------- + str + Absolute path of the written file. + """ + os.makedirs(output_root, exist_ok=True) + filename = 'ccpp_{}_{}_cap.F90'.format(suite_name, group_name) + out_path = os.path.join(output_root, filename) + + lines = _generate_group_cap( + suite_name, group_name, resolved_group, host_dict, trace=trace, + ) + with open_if_changed(out_path, logger=logger) as fh: + fh.write('\n'.join(lines) + '\n') + return out_path diff --git a/capgen/generator/host_cap.py b/capgen/generator/host_cap.py new file mode 100644 index 00000000..f790b318 --- /dev/null +++ b/capgen/generator/host_cap.py @@ -0,0 +1,1194 @@ +#!/usr/bin/env python3 + +"""Generate the static API module ``_ccpp_cap.F90``. + +The static API module is generated once per build (not per suite) and +provides the canonical public entry points that a host model calls: + + - ``ccpp_register`` + - ``ccpp_init`` + - ``ccpp_physics_init``, ``ccpp_physics_timestep_init``, + ``ccpp_physics_run``, ``ccpp_physics_timestep_final``, + ``ccpp_physics_final`` + - ``ccpp_final`` + +Each entry point dispatches by ``suite_name`` (and ``group_name`` for the +physics-phase calls) to the corresponding suite cap subroutine. + +The module also exposes five suite-introspection routines so a host +can query at runtime what is compiled into the API: + + - ``ccpp_physics_suite_list(suites)`` — names of all compiled-in suites. + - ``ccpp_physics_suite_part_list(suite_name, part_list, errmsg, errflg)`` + — group ("part") names belonging to *suite_name*. + - ``ccpp_physics_suite_schemes(suite_name, scheme_list, errmsg, errflg)`` + — scheme module names that compose *suite_name* (deduped across phases). + - ``ccpp_physics_suite_variables(suite_name, variable_list, errmsg, + errflg, [input_vars], [output_vars])`` — flat leaf standard names the + host exchanges with the suite. ``input_vars`` selects intent in/inout + (skipping ``protected`` host vars); ``output_vars`` selects intent + out/inout. Both default ``.true.`` (the union, deduplicated). + - ``ccpp_physics_suite_host_data(suite_name, variable_list, errmsg, + errflg, [input_vars], [output_vars])`` — same intent-filter semantics, + but DDT-leaf standard names are collapsed to the standard name of the + top-level DDT instance the host owns. Plain (non-DDT) leaves appear + individually, just like in ``..._suite_variables``. + +The suite-variables / suite-host-data routines exclude control variables +(those passed via the framework signature) — hosts already know their own +control table, and excluding them keeps the lists focused on the data the +host has to read/write to interface with the suite. +""" + +import logging +import os +from typing import Dict, List, Optional, Set, Tuple + +from metadata.parse_tools import CCPPError, open_if_changed +from metadata.variable_resolver import SchemeStore +from generator.suite_resolver import ( + ResolvedArg, + SuiteResolution, + iter_phase_calls, + iter_phase_subcycles, +) +from metadata.variable_resolver import HostVarEntry +from generator.trace import ( + emit_module_gate, + emit_trace_block, + ensure_error_unit_use, +) +from generator.suite_cap import ( + _all_suite_scheme_names, + _schemes_with_register, + _suite_ctrl_args_for_phase, + _suite_extra_ctrl_entries_for_phase, + _PHYSICS_PHASES, + _CONST_MOD, + _CONST_DDT, + _CONST_PROP_TYPE, +) +from generator.group_cap import ( + _active_std_names, + _ctrl_entries_for_signature, + _ctrl_intent_for, + _ctrl_local, + _fortran_type_str, + _dim_decl, + _instance_local, + _intent_clause, +) + +_INDENT = ' ' + +# Message bodied into stubbed suite-introspection routines when the +# generator was invoked with ``--no-host-introspection``. Kept here so +# every introspection routine produces the same wording and tests can +# assert against a single constant. +_INTROSPECTION_DISABLED_MSG = ( + 'suite introspection disabled at code-generation time; ' + 'regenerate caps without --no-host-introspection' +) + + +def _emit_introspection_stub_body( + routine_name: str, + list_arg_name: str, + indent: str, +) -> List[str]: + """Emit the stub body shared by the four errflg-bearing introspection + routines (``suite_part_list``, ``suite_schemes``, ``suite_variables``, + ``suite_host_data``). + + Sets ``errflg = 1`` and ``errmsg`` to the canonical disabled-message + prefixed with *routine_name*, then allocates *list_arg_name* to a + zero-length array so callers can safely ``size()`` / iterate without + a NULL-deref crash. Indentation level *indent* matches the body + block of the calling routine. + """ + lines: List[str] = [] + lines.append("{}errmsg = '{}: {}'".format( + indent, routine_name, _INTROSPECTION_DISABLED_MSG, + )) + lines.append('{}errflg = 1'.format(indent)) + lines.append('{}allocate({}(0))'.format(indent, list_arg_name)) + return lines + + +######################################################################## +# Helpers +######################################################################## + +def _all_ctrl_args_for_phase( + suite_resolutions: List[SuiteResolution], + phase: str, +) -> List[ResolvedArg]: + """Return the union of direct scheme control args across all suites for *phase*. + + Deduplicated by standard_name, first-seen order. + """ + seen: Dict[str, ResolvedArg] = {} + for suite_resolution in suite_resolutions: + for arg in _suite_ctrl_args_for_phase(suite_resolution, phase): + if arg.standard_name not in seen: + seen[arg.standard_name] = arg + return list(seen.values()) + + +def _all_extra_ctrl_entries_for_phase( + suite_resolutions: List[SuiteResolution], + phase: str, + ctrl_std_names: Set[str], + host_dict, +) -> List[HostVarEntry]: + """Return extra HostVarEntry objects (state indexing / dim subscripts) needed + by any suite-group for *phase* but not already in *ctrl_std_names*. + """ + if host_dict is None: + return [] + seen = set(ctrl_std_names) + result: Dict[str, HostVarEntry] = {} + for suite_resolution in suite_resolutions: + for entry in _suite_extra_ctrl_entries_for_phase(suite_resolution, phase, seen, host_dict): + if entry.standard_name not in seen and entry.standard_name not in result: + result[entry.standard_name] = entry + return list(result.values()) + + +######################################################################## +# Helpers — suite-introspection +######################################################################## + +def _emit_var_set_loop( + list_name: str, + items: List[str], + indent: str, + allocate: bool = True, +) -> List[str]: + """Emit ``allocate((N))`` followed by per-element assignments. + + Items are emitted as 1-based Fortran assignments: ``(i) = ''``. + Used by every introspection routine that returns a string list. + + >>> _emit_var_set_loop('x', ['a', 'b'], ' ') + [' allocate(x(2))', " x(1) = 'a'", " x(2) = 'b'"] + >>> _emit_var_set_loop('x', [], ' ') + [' allocate(x(0))'] + """ + lines = [] + if allocate: + lines.append('{}allocate({}({}))'.format(indent, list_name, len(items))) + for i, item in enumerate(items): + lines.append("{}{}({}) = '{}'".format(indent, list_name, i + 1, item)) + return lines + + +def _build_local_to_std_top_level_map(host_dict) -> Dict[str, str]: + """Build local_name → standard_name map for top-level host_dict entries. + + Only entries whose ``access_path`` does not contain ``'%'`` are + included — those are top-level DDT instances or plain (non-DDT) + leaves. Used to collapse a flattened DDT-leaf back to the standard + name of the DDT instance the host owns. + """ + if not host_dict: + return {} + return { + entry.local_name: entry.standard_name + for entry in host_dict.values() + if '%' not in entry.access_path + } + + +def _arg_top_level_name( + arg: ResolvedArg, + local_to_std: Dict[str, str], +) -> str: + """Return the top-level standard name for *arg* (DDT-collapsed view). + + For a plain leaf or a directly-referenced DDT instance, returns + ``arg.standard_name`` unchanged. For a DDT-leaf access path like + ``phys_state(instance_number)%t``, parses the root local name + (``phys_state``), strips any subscript, and looks up the standard + name of the host_dict entry whose ``local_name`` matches. + + Falls back to ``arg.standard_name`` if the lookup fails (which would + indicate inconsistent metadata, but produces a well-defined output). + """ + if arg.host_entry is None: + return arg.standard_name + ap = arg.host_entry.access_path + if '%' not in ap: + return arg.standard_name + root = ap.split('%', 1)[0] + paren = root.find('(') + if paren >= 0: + root = root[:paren] + return local_to_std.get(root, arg.standard_name) + + +def _collect_host_io( + suite_resolution: SuiteResolution, + host_dict=None, + collapse_ddts: bool = False, +) -> Tuple[List[str], List[str]]: + """Collect (inputs, outputs) standard names for the introspection routines. + + Walks every phase of every group of *suite_resolution*. Includes scheme args from + every ``source`` category EXCEPT ``'suite'`` — suite-owned vars are + internal data flow between schemes (one scheme writes them, another + reads them) and are not part of the host-facing variable list. + + Concretely, the returned lists include: + + * ``source='host'`` — host metadata vars. + * ``source='control'`` — control vars (errmsg, errflg, …) when they + appear as scheme args (not when they're framework-injected dummies). + * ``source='constituent'`` — both auto-resolved base/tendency + constituents and direct framework-array references + (``ccpp_constituents`` / ``ccpp_constituent_tendencies`` / etc.) and + register-phase ``ccpp_constituent_properties_t`` args. + + *inputs* : intent in ``('in', 'inout')`` and not protected (the + protected check is host-only; constituent / control args + have no protected attribute). + *outputs* : intent in ``('out', 'inout')``. + + When *collapse_ddts* is true, DDT-leaf standard names are mapped to + the standard name of the top-level DDT instance via *host_dict*. + If *host_dict* is ``None`` while *collapse_ddts* is true, the lookup + map is empty and every leaf falls back to its own standard name — + correct in scenarios with no DDT instances (e.g. minimal unit tests). + Both lists are returned sorted alphabetically for deterministic output. + + This matches the behavior of the original capgen's + ``ccpp_physics_suite_variables`` so host comparison tests round-trip + cleanly. + """ + local_to_std = _build_local_to_std_top_level_map(host_dict) if collapse_ddts else {} + + def _collapse_std(std_name: str) -> str: + """Map a CCPP standard name to its DDT-collapsed counterpart. + + For free host variables the standard name maps to itself; for a + DDT-component entry the result is the standard name of the + top-level instance. When ``collapse_ddts`` is False or the + std_name isn't in ``host_dict``, returns the input unchanged. + """ + if not collapse_ddts or host_dict is None: + return std_name + entry = host_dict.get(std_name) + if entry is None or '%' not in entry.access_path: + return std_name + root = entry.access_path.split('%', 1)[0] + paren = root.find('(') + if paren >= 0: + root = root[:paren] + return local_to_std.get(root, std_name) + + inputs: Set[str] = set() + outputs: Set[str] = set() + for group in suite_resolution.groups: + for items in group.phase_calls.values(): + # Subcycle loop bounds named by a CCPP standard name (e.g. + # ````) are pure + # inputs from the host — the host supplies the value and the + # generated cap reads it as a do-loop bound. Every nesting + # level contributes a (potentially different) bound, so walk + # the full subcycle tree. Without this, the host's + # compile-time bookkeeping (ccpp_physics_suite_variables / + # _suite_host_data) would silently omit required inputs. + for subcycle in iter_phase_subcycles(items): + if subcycle.loop_std_name: + inputs.add(_collapse_std(subcycle.loop_std_name)) + for call in iter_phase_calls(items): + for arg in call.args: + # Suite-owned vars are internal scheme-to-scheme + # plumbing and not part of the host-facing surface. + if arg.source == 'suite': + continue + name = ( + _arg_top_level_name(arg, local_to_std) + if collapse_ddts + else arg.standard_name + ) + if arg.intent in ('in', 'inout'): + # Only host args have a meaningful protected flag. + if not (arg.host_entry and arg.host_entry.protected): + inputs.add(name) + if arg.intent in ('out', 'inout'): + outputs.add(name) + # Framework-constituent dim references (e.g. + # number_of_ccpp_constituents as the trailing dim of + # ccpp_constituents) appear in the inputs list even + # though they don't have a dedicated scheme arg. + # Tracked on a dedicated field — these names are + # NOT in host_dict and must not be USE'd, so they + # don't live on used_dim_std_names. + for dim_std in arg.used_const_dim_std_names: + inputs.add(dim_std) + # Active-expression references (e.g. + # ``active = (flag_indicating_...)`` on a host var) + # are pure inputs: the host provides the flag so the + # suite knows whether the active arg is present. + # Without this, flags that *aren't* used as a direct + # scheme arg silently fall out of the introspection + # input list. + for active_std in _active_std_names(arg.active): + # Skip Fortran literals captured by the + # active-name tokenizer (e.g. ``.true.``). + if host_dict is not None and active_std in host_dict: + inputs.add(_collapse_std(active_std)) + return sorted(inputs), sorted(outputs) + + +######################################################################## +# Entry point generators +######################################################################## + +def _register_subroutine(suite_names: List[str], host_dict=None) -> List[str]: + """Generate ``ccpp_register`` (mandatory entry point). + + Always emitted with the minimal lifecycle signature + ``(suite_name, ccpp_error_code, ccpp_error_message, + [instance_number, number_of_instances])``. The instance pair is + forwarded to ``_register`` only when the host declares it. + The body dispatches to ``_register`` for every known suite. Each + suite's register routine is responsible for allocating its state array + and DDT instance array, calling its register-phase scheme entrypoints, + and transitioning suite state to ``REGISTERED`` — even if the suite has + no register-providing schemes (the state transition still fires). + """ + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + + inst_local = _instance_local(host_dict) + ninst_entry = host_dict.get('number_of_instances') if host_dict else None + ninst_local = ninst_entry.local_name if ninst_entry else None + + errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') or 'errflg' + errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') or 'errmsg' + suite_name_entry = host_dict.get('suite_name') if host_dict else None + suite_name_local = ( + suite_name_entry.local_name if suite_name_entry else 'suite_name' + ) + + sig_args = [suite_name_local, errflg_local, errmsg_local] + suite_call_args = [] + if inst_local: + sig_args.append(inst_local) + suite_call_args.append(inst_local) + if ninst_local: + sig_args.append(ninst_local) + suite_call_args.append(ninst_local) + suite_call_args += [errmsg_local, errflg_local] + + lines: List[str] = [''] + lines.append('{}subroutine ccpp_register({})'.format(i1, ', '.join(sig_args))) + lines.append('') + lines.append('{}character(len=*), intent(in) :: {}'.format(i2, suite_name_local)) + lines.append('{}integer, intent(out) :: {}'.format(i2, errflg_local)) + lines.append('{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local)) + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + if ninst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, ninst_local)) + trace_entries = [suite_name_entry] if suite_name_entry else [] + extra_in = [ninst_local] if ninst_local else None + trace_lines = emit_trace_block( + 'ccpp_register', trace_entries, i2, + instance_local=inst_local, extra_in_names=extra_in, + ) + if trace_lines: + lines.append('') + lines.extend(trace_lines) + lines += [ + '', + "{}{} = ''".format(i2, errmsg_local), + '{}{} = 0'.format(i2, errflg_local), + '', + '{}select case(trim({}))'.format(i2, suite_name_local), + ] + for sname in suite_names: + lines.append("{}case('{}')".format(i2, sname)) + lines.append('{}call {}_register({})'.format( + i3, sname, ', '.join(suite_call_args) + )) + lines += [ + '{}case default'.format(i2), + '{}{} = 1'.format(i3, errflg_local), + "{}{} = 'ccpp_register: unknown suite: ' // trim({})".format( + i3, errmsg_local, suite_name_local + ), + '{}end select'.format(i2), + '', + '{}end subroutine ccpp_register'.format(i1), + ] + return lines + + +def _init_subroutine(suite_names: List[str], host_dict=None) -> List[str]: + """Generate ``ccpp_init`` (minimal lifecycle signature). + + Signature: ``(suite_name, ccpp_error_code, ccpp_error_message, + [instance_number, number_of_instances])``. The instance pair is + forwarded to ``_init`` only when the host declares it. + """ + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + + inst_local = _instance_local(host_dict) + ninst_entry = host_dict.get('number_of_instances') if host_dict else None + ninst_local = ninst_entry.local_name if ninst_entry else None + + errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') or 'errflg' + errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') or 'errmsg' + suite_name_entry = host_dict.get('suite_name') if host_dict else None + suite_name_local = ( + suite_name_entry.local_name if suite_name_entry else 'suite_name' + ) + + sig_args = [suite_name_local, errflg_local, errmsg_local] + suite_call_args: List[str] = [] + if inst_local: + sig_args.append(inst_local) + suite_call_args.append(inst_local) + if ninst_local: + sig_args.append(ninst_local) + suite_call_args.append(ninst_local) + suite_call_args += [errmsg_local, errflg_local] + + lines: List[str] = [''] + lines.append('{}subroutine ccpp_init({})'.format(i1, ', '.join(sig_args))) + lines.append('') + lines.append('{}character(len=*), intent(in) :: {}'.format(i2, suite_name_local)) + lines.append('{}integer, intent(out) :: {}'.format(i2, errflg_local)) + lines.append('{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local)) + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + if ninst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, ninst_local)) + trace_entries = [suite_name_entry] if suite_name_entry else [] + extra_in = [ninst_local] if ninst_local else None + trace_lines = emit_trace_block( + 'ccpp_init', trace_entries, i2, + instance_local=inst_local, extra_in_names=extra_in, + ) + if trace_lines: + lines.append('') + lines.extend(trace_lines) + lines += [ + '', + "{}{} = ''".format(i2, errmsg_local), + '{}{} = 0'.format(i2, errflg_local), + '', + '{}select case(trim({}))'.format(i2, suite_name_local), + ] + for sname in suite_names: + lines.append("{}case('{}')".format(i2, sname)) + lines.append('{}call {}_init({})'.format( + i3, sname, ', '.join(suite_call_args) + )) + lines += [ + '{}case default'.format(i2), + '{}{} = 1'.format(i3, errflg_local), + "{}{} = 'ccpp_init: unknown suite: ' // trim({})".format( + i3, errmsg_local, suite_name_local + ), + '{}end select'.format(i2), + '', + '{}end subroutine ccpp_init'.format(i1), + ] + return lines + + +def _final_subroutine(suite_names: List[str], host_dict=None) -> List[str]: + """Generate ``ccpp_final`` (lifecycle signature). + + Signature: ``(suite_name, ccpp_error_code, ccpp_error_message, + [instance_number, number_of_instances])``. ``number_of_instances`` + is carried for API symmetry with ``ccpp_register`` / ``ccpp_init`` + even though the framework does not need it at final time. + """ + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + + inst_local = _instance_local(host_dict) + ninst_entry = host_dict.get('number_of_instances') if host_dict else None + ninst_local = ninst_entry.local_name if ninst_entry else None + + errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') or 'errflg' + errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') or 'errmsg' + suite_name_entry = host_dict.get('suite_name') if host_dict else None + suite_name_local = ( + suite_name_entry.local_name if suite_name_entry else 'suite_name' + ) + + sig_args = [suite_name_local, errflg_local, errmsg_local] + suite_call_args: List[str] = [] + if inst_local: + sig_args.append(inst_local) + suite_call_args.append(inst_local) + if ninst_local: + sig_args.append(ninst_local) + suite_call_args.append(ninst_local) + suite_call_args += [errmsg_local, errflg_local] + + lines: List[str] = [''] + lines.append('{}subroutine ccpp_final({})'.format(i1, ', '.join(sig_args))) + lines.append('') + lines.append('{}character(len=*), intent(in) :: {}'.format(i2, suite_name_local)) + lines.append('{}integer, intent(out) :: {}'.format(i2, errflg_local)) + lines.append('{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local)) + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + if ninst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, ninst_local)) + trace_entries = [suite_name_entry] if suite_name_entry else [] + extra_in = [ninst_local] if ninst_local else None + trace_lines = emit_trace_block( + 'ccpp_final', trace_entries, i2, + instance_local=inst_local, extra_in_names=extra_in, + ) + if trace_lines: + lines.append('') + lines.extend(trace_lines) + lines += [ + '', + "{}{} = ''".format(i2, errmsg_local), + '{}{} = 0'.format(i2, errflg_local), + '', + '{}select case(trim({}))'.format(i2, suite_name_local), + ] + for sname in suite_names: + lines.append("{}case('{}')".format(i2, sname)) + lines.append('{}call {}_final({})'.format( + i3, sname, ', '.join(suite_call_args) + )) + lines += [ + '{}case default'.format(i2), + '{}{} = 1'.format(i3, errflg_local), + "{}{} = 'ccpp_final: unknown suite: ' // trim({})".format( + i3, errmsg_local, suite_name_local + ), + '{}end select'.format(i2), + '', + '{}end subroutine ccpp_final'.format(i1), + ] + return lines + + +def _physics_subroutine( + phase: str, + suite_names: List[str], + suite_resolutions: List[SuiteResolution], + host_dict=None, +) -> List[str]: + """Generate one ``ccpp_physics_`` dispatch subroutine. + + The formal argument list is derived entirely from the host's ``type=control`` + metadata table. ``suite_name`` drives the top-level dispatch; ``group_name`` + (if present in the control table) is forwarded to the suite cap dispatch. + """ + sub_name = 'ccpp_physics_{}'.format(phase) + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + + # Full control arg list (all control vars from host_dict). + ctrl_entries = _ctrl_entries_for_signature(host_dict) + ctrl_local_names = [e.local_name for e in ctrl_entries] + + # Args forwarded to suite cap (all ctrl vars except suite_name). + suite_ctrl_entries = _ctrl_entries_for_signature(host_dict, exclude={'suite_name'}) + suite_ctrl_local = [e.local_name for e in suite_ctrl_entries] + + # Local name for suite_name (used in select case dispatch). + suite_name_entry = next( + (e for e in ctrl_entries if e.standard_name == 'suite_name'), None + ) + suite_name_local = suite_name_entry.local_name if suite_name_entry else 'suite_name' + + lines = [''] + + # Subroutine signature. + if ctrl_local_names: + lines.append('{}subroutine {}( &'.format(i1, sub_name)) + for i, lname in enumerate(ctrl_local_names): + sep = ', &' if i < len(ctrl_local_names) - 1 else ')' + lines.append('{} {}{}'.format(i1, lname, sep)) + else: + lines.append('{}subroutine {}()'.format(i1, sub_name)) + + # Dummy declarations. + lines.append('') + for entry in ctrl_entries: + # Character control dummies always use len=* so the host's specific + # length doesn't propagate into the generated signature. + kind = 'len=*' if entry.type.strip().lower() == 'character' else entry.kind + t = _fortran_type_str(entry.type, kind) + dim = _dim_decl(entry.dimensions) + intent = _intent_clause(_ctrl_intent_for(entry.standard_name)) + lines.append( + '{}{}{}{} :: {}'.format(i2, t, intent, dim, entry.local_name) + ) + + # Trace block: every intent(in)/inout control dummy is referenced so + # strict compilers don't flag any of them as unused. + trace_lines = emit_trace_block(sub_name, ctrl_entries, i2) + if trace_lines: + lines.append('') + lines.extend(trace_lines) + + lines.append('') + + # Initialize error reporting vars before any work. + errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') + errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') + if errflg_local and errmsg_local: + lines.append("{}{} = ''".format(i2, errmsg_local)) + lines.append('{}{} = 0'.format(i2, errflg_local)) + lines.append('') + + # select case(suite_name) dispatch. + lines.append('{}select case(trim({}))'.format(i2, suite_name_local)) + for sname in suite_names: + lines.append("{}case('{}')".format(i2, sname)) + cap_sub = '{}_physics_{}'.format(sname, phase) + if suite_ctrl_local: + lines.append('{}call {}( &'.format(i3, cap_sub)) + for i, carg in enumerate(suite_ctrl_local): + sep = ', &' if i < len(suite_ctrl_local) - 1 else ')' + lines.append('{} {}{}'.format(i3, carg, sep)) + else: + lines.append('{}call {}()'.format(i3, cap_sub)) + # case default: unknown suite name is a runtime error (not silent + # fall-through). Skip emission only when the host doesn't carry the + # standard error-reporting control vars — without somewhere to write + # the message, there is nothing meaningful to do here. + if errflg_local and errmsg_local: + lines.append('{}case default'.format(i2)) + lines.append('{}{} = 1'.format(i3, errflg_local)) + lines.append( + "{}{} = '{}: unknown suite: ' // trim({})".format( + i3, errmsg_local, sub_name, suite_name_local, + ) + ) + lines.append('{}end select'.format(i2)) + + lines.append('') + lines.append('{}end subroutine {}'.format(i1, sub_name)) + return lines + + +######################################################################## +# Suite-introspection subroutines +######################################################################## + +def _suite_list_subroutine( + suite_names: List[str], + stub_body: bool = False, +) -> List[str]: + """Generate ``ccpp_physics_suite_list(suites)``. + + Allocates ``suites`` to the number of compiled-in suites and assigns + each entry to the suite name as a literal string. + + When *stub_body* is true, emit a stub: write a clear message to + ``error_unit`` and return an empty list. ``ccpp_physics_suite_list`` + has no errflg/errmsg arguments, so ``error_unit`` is the only + available error channel. The module-level ``use iso_fortran_env`` + that this references is added by ``_generate_host_cap`` when + ``no_host_introspection`` is on. + """ + i1 = _INDENT + i2 = _INDENT * 2 + + lines: List[str] = [''] + lines.append('{}subroutine ccpp_physics_suite_list(suites)'.format(i1)) + lines.append('') + lines.append( + '{}character(len=*), allocatable, intent(out) :: suites(:)'.format(i2) + ) + lines.append('') + if stub_body: + lines.append( + "{}write(error_unit, '(a)') 'ccpp_physics_suite_list: ' &" + .format(i2) + ) + lines.append( + "{} // '{}'".format(i2, _INTROSPECTION_DISABLED_MSG) + ) + lines.append('{}allocate(suites(0))'.format(i2)) + else: + lines.extend(_emit_var_set_loop('suites', suite_names, i2)) + lines.append('') + lines.append('{}end subroutine ccpp_physics_suite_list'.format(i1)) + return lines + + +def _suite_part_list_subroutine( + suite_names: List[str], + suite_resolutions: List[SuiteResolution], + stub_body: bool = False, +) -> List[str]: + """Generate ``ccpp_physics_suite_part_list(suite_name, part_list, errmsg, errflg)``. + + Dispatches by ``suite_name`` and returns the group ("part") names of + that suite in declaration order. ``case default`` sets ``errflg=1`` + and writes a message naming the unknown suite. + """ + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + + lines: List[str] = [''] + lines.append( + '{}subroutine ccpp_physics_suite_part_list( &'.format(i1) + ) + lines.append('{} suite_name, part_list, errmsg, errflg)'.format(i1)) + lines.append('') + lines.append( + '{}character(len=*), intent(in) :: suite_name'.format(i2) + ) + lines.append( + '{}character(len=*), allocatable, intent(out) :: part_list(:)'.format(i2) + ) + lines.append( + '{}character(len=*), intent(out) :: errmsg'.format(i2) + ) + lines.append( + '{}integer, intent(out) :: errflg'.format(i2) + ) + lines.append('') + if stub_body: + lines.extend( + _emit_introspection_stub_body( + 'ccpp_physics_suite_part_list', 'part_list', i2, + ) + ) + else: + lines.append("{}errmsg = ''".format(i2)) + lines.append('{}errflg = 0'.format(i2)) + lines.append('') + lines.append('{}select case (trim(suite_name))'.format(i2)) + for sname, suite_resolution in zip(suite_names, suite_resolutions): + groups = [g.group_name for g in suite_resolution.groups] + lines.append("{}case ('{}')".format(i2, sname)) + lines.extend(_emit_var_set_loop('part_list', groups, i3)) + lines.append('{}case default'.format(i2)) + lines.append('{}errflg = 1'.format(i3)) + lines.append( + "{}errmsg = 'ccpp_physics_suite_part_list: unknown suite: ' " + "// trim(suite_name)".format(i3) + ) + lines.append('{}end select'.format(i2)) + lines.append('') + lines.append('{}end subroutine ccpp_physics_suite_part_list'.format(i1)) + return lines + + +def _suite_schemes_subroutine( + suite_names: List[str], + suite_resolutions: List[SuiteResolution], + stub_body: bool = False, +) -> List[str]: + """Generate ``ccpp_physics_suite_schemes(suite_name, scheme_list, errmsg, errflg)``. + + Returns the unique scheme names that compose *suite_name*, deduped + across all phases and groups, sorted alphabetically. + """ + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + + lines: List[str] = [''] + lines.append( + '{}subroutine ccpp_physics_suite_schemes( &'.format(i1) + ) + lines.append('{} suite_name, scheme_list, errmsg, errflg)'.format(i1)) + lines.append('') + lines.append( + '{}character(len=*), intent(in) :: suite_name'.format(i2) + ) + lines.append( + '{}character(len=*), allocatable, intent(out) :: scheme_list(:)'.format(i2) + ) + lines.append( + '{}character(len=*), intent(out) :: errmsg'.format(i2) + ) + lines.append( + '{}integer, intent(out) :: errflg'.format(i2) + ) + lines.append('') + if stub_body: + lines.extend( + _emit_introspection_stub_body( + 'ccpp_physics_suite_schemes', 'scheme_list', i2, + ) + ) + else: + lines.append("{}errmsg = ''".format(i2)) + lines.append('{}errflg = 0'.format(i2)) + lines.append('') + lines.append('{}select case (trim(suite_name))'.format(i2)) + for sname, suite_resolution in zip(suite_names, suite_resolutions): + schemes = sorted({ + call.scheme_name + for group in suite_resolution.groups + for items in group.phase_calls.values() + for call in iter_phase_calls(items) + }) + lines.append("{}case ('{}')".format(i2, sname)) + lines.extend(_emit_var_set_loop('scheme_list', schemes, i3)) + lines.append('{}case default'.format(i2)) + lines.append('{}errflg = 1'.format(i3)) + lines.append( + "{}errmsg = 'ccpp_physics_suite_schemes: unknown suite: ' " + "// trim(suite_name)".format(i3) + ) + lines.append('{}end select'.format(i2)) + lines.append('') + lines.append('{}end subroutine ccpp_physics_suite_schemes'.format(i1)) + return lines + + +def _suite_io_subroutine( + suite_names: List[str], + suite_resolutions: List[SuiteResolution], + host_dict=None, + collapse_ddts: bool = False, + stub_body: bool = False, +) -> List[str]: + """Generate ``ccpp_physics_suite_variables`` or ``ccpp_physics_suite_host_data``. + + The two routines share the same Fortran shape — ``select case`` on + ``suite_name`` with one branch per suite and three nested branches + keyed on ``input_vars`` / ``output_vars`` — so they are emitted by + one helper. *collapse_ddts* controls both the routine name and + whether DDT-leaves get mapped to their top-level DDT instance. + + Optional ``input_vars``/``output_vars`` default to ``.true.``; + when both are ``.true.`` the union of inputs and outputs is returned + (deduplicated). When both are ``.false.`` an empty list is returned. + """ + sub_name = ( + 'ccpp_physics_suite_host_data' + if collapse_ddts + else 'ccpp_physics_suite_variables' + ) + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + i4 = _INDENT * 4 + + lines: List[str] = [''] + lines.append('{}subroutine {}( &'.format(i1, sub_name)) + lines.append( + '{} suite_name, variable_list, errmsg, errflg, &'.format(i1) + ) + lines.append('{} input_vars, output_vars)'.format(i1)) + lines.append('') + lines.append( + '{}character(len=*), intent(in) :: suite_name'.format(i2) + ) + lines.append( + '{}character(len=*), allocatable, intent(out) :: variable_list(:)'.format(i2) + ) + lines.append( + '{}character(len=*), intent(out) :: errmsg'.format(i2) + ) + lines.append( + '{}integer, intent(out) :: errflg'.format(i2) + ) + lines.append( + '{}logical, optional, intent(in) :: input_vars'.format(i2) + ) + lines.append( + '{}logical, optional, intent(in) :: output_vars'.format(i2) + ) + lines.append('') + if stub_body: + # Stubbed body: set errflg + clear errmsg and allocate an empty + # list. ``input_vars`` / ``output_vars`` are intentionally left + # unreferenced — they're declared ``intent(in), optional`` so + # compilers may warn about the unused dummy arg, but warning is + # the right outcome: the host built against introspection and + # is calling with filter flags that no longer matter. + lines.append('') + lines.extend( + _emit_introspection_stub_body(sub_name, 'variable_list', i2) + ) + else: + lines.append('{}logical :: input_vars_use'.format(i2)) + lines.append('{}logical :: output_vars_use'.format(i2)) + lines.append('') + lines.append("{}errmsg = ''".format(i2)) + lines.append('{}errflg = 0'.format(i2)) + lines.append('') + lines.append('{}if (present(input_vars)) then'.format(i2)) + lines.append('{}input_vars_use = input_vars'.format(i3)) + lines.append('{}else'.format(i2)) + lines.append('{}input_vars_use = .true.'.format(i3)) + lines.append('{}end if'.format(i2)) + lines.append('{}if (present(output_vars)) then'.format(i2)) + lines.append('{}output_vars_use = output_vars'.format(i3)) + lines.append('{}else'.format(i2)) + lines.append('{}output_vars_use = .true.'.format(i3)) + lines.append('{}end if'.format(i2)) + lines.append('') + lines.append('{}select case (trim(suite_name))'.format(i2)) + for sname, suite_resolution in zip(suite_names, suite_resolutions): + inputs, outputs = _collect_host_io(suite_resolution, host_dict, collapse_ddts) + union = sorted(set(inputs) | set(outputs)) + lines.append("{}case ('{}')".format(i2, sname)) + lines.append('{}if (input_vars_use .and. output_vars_use) then'.format(i3)) + lines.extend(_emit_var_set_loop('variable_list', union, i4)) + lines.append('{}else if (input_vars_use) then'.format(i3)) + lines.extend(_emit_var_set_loop('variable_list', inputs, i4)) + lines.append('{}else if (output_vars_use) then'.format(i3)) + lines.extend(_emit_var_set_loop('variable_list', outputs, i4)) + lines.append('{}else'.format(i3)) + lines.append('{}allocate(variable_list(0))'.format(i4)) + lines.append('{}end if'.format(i3)) + lines.append('{}case default'.format(i2)) + lines.append('{}errflg = 1'.format(i3)) + lines.append( + "{}errmsg = '{}: unknown suite: ' " + "// trim(suite_name)".format(i3, sub_name) + ) + lines.append('{}end select'.format(i2)) + lines.append('') + lines.append('{}end subroutine {}'.format(i1, sub_name)) + return lines + + +######################################################################## +# Module generator +######################################################################## + +def _generate_host_cap( + host_name: str, + suite_names: List[str], + suite_resolutions: List[SuiteResolution], + host_dict=None, + scheme_store: Optional[SchemeStore] = None, + no_host_introspection: bool = False, + trace: bool = False, +) -> List[str]: + """Generate the full ``_ccpp_cap.F90`` module source lines. + + Parameters + ---------- + host_name : str + Host identifier; drives the emitted module name + (``module _ccpp_cap``) and the comment header. + suite_names : list of str + Suite names in order. + suite_resolutions : list of SuiteResolution + Parallel to suite_names. + host_dict : dict, optional + Flat host+control dictionary. When provided, ``number_of_instances`` + is threaded through ``ccpp_init`` for multi-instance support. + scheme_store : SchemeStore, optional + When provided, used to determine which suites have at least one + scheme with a ``register`` phase. Only those suites contribute a + ``case`` arm to ``ccpp_register``, and ``ccpp_register`` itself is + only emitted if any suite qualifies. When omitted, no suite is + treated as having a register phase. + + Returns + ------- + list of str (no trailing newlines) + """ + if len(suite_names) != len(suite_resolutions): + raise CCPPError( + 'suite_names and suite_resolutions must have the same length' + ) + + mod_name = '{}_ccpp_cap'.format(host_name) + + lines: List[str] = [] + lines.append( + '! {}.F90 -- generated by ccpp_capgen, do not edit'.format(mod_name) + ) + lines.append('module {}'.format(mod_name)) + lines.append('') + + # Collect USE lines into a list so the trace helper can guarantee + # ``error_unit`` is present. The trace block writes to error_unit + # and is emitted in every cap subroutine (gated by the module + # ``trace`` parameter), so the USE is now unconditional. Replaces + # an earlier --no-host-introspection-only emission. + use_lines: List[str] = [] + for sname in suite_names: + suite_cap_mod = 'ccpp_{}_cap'.format(sname) + suite_subs = [] + suite_subs.append('{}_register'.format(sname)) + suite_subs.append('{}_init'.format(sname)) + for phase in _PHYSICS_PHASES: + suite_subs.append('{}_physics_{}'.format(sname, phase)) + suite_subs.append('{}_final'.format(sname)) + syms = ', '.join(suite_subs) + use_lines.append('{}use {}, only: {}'.format(_INDENT, suite_cap_mod, syms)) + ensure_error_unit_use(use_lines, _INDENT) + lines.extend(use_lines) + + # Re-export the host-facing constituent API + the constituent object so + # host code can do ``use _ccpp_cap, only: ...`` for *everything* + # it needs from CCPP. Mirrors original capgen, which put all of these + # on the generated host cap module. Only emitted when any suite uses + # constituent state (the ccpp_host_constituents module is only emitted + # in that case too). + uses_consts = any( + suite_resolution.uses_constituents or suite_resolution.constituent_register_calls + for suite_resolution in suite_resolutions + ) + constituent_pub_syms = [ + 'ccpp_model_constituents_obj', + 'ccpp_register_constituents', + 'ccpp_initialize_constituents', + 'ccpp_is_scheme_constituent', + 'ccpp_number_constituents', + 'ccpp_gather_constituents', + 'ccpp_update_constituents', + 'ccpp_const_get_index', + 'ccpp_constituents_array', + 'ccpp_advected_constituents_array', + 'ccpp_model_const_properties', + 'ccpp_deallocate_dynamic_constituents', + ] + if uses_consts: + lines.append('{}use ccpp_host_constituents, only: &'.format(_INDENT)) + for i, sym in enumerate(constituent_pub_syms): + sep = ', &' if i < len(constituent_pub_syms) - 1 else '' + lines.append('{}{}{}'.format(_INDENT * 2, sym, sep)) + + lines.append('') + lines.append('{}implicit none'.format(_INDENT)) + lines.append('{}private'.format(_INDENT)) + lines.append('') + + # Public declarations. ccpp_register is mandatory. + pub_subs = [] + pub_subs.append('ccpp_register') + pub_subs.append('ccpp_init') + for phase in _PHYSICS_PHASES: + pub_subs.append('ccpp_physics_{}'.format(phase)) + pub_subs.append('ccpp_final') + pub_subs.append('ccpp_physics_suite_list') + pub_subs.append('ccpp_physics_suite_part_list') + pub_subs.append('ccpp_physics_suite_schemes') + pub_subs.append('ccpp_physics_suite_variables') + pub_subs.append('ccpp_physics_suite_host_data') + if uses_consts: + pub_subs.extend(constituent_pub_syms) + for sub in pub_subs: + lines.append('{}public :: {}'.format(_INDENT, sub)) + + lines.append('') + lines.extend(emit_module_gate(trace, _INDENT)) + lines.append('') + lines.append('contains') + + # Subroutines. + lines.extend(_register_subroutine(suite_names, host_dict)) + lines.extend(_init_subroutine(suite_names, host_dict)) + for phase in _PHYSICS_PHASES: + lines.extend(_physics_subroutine(phase, suite_names, suite_resolutions, host_dict)) + lines.extend(_final_subroutine(suite_names, host_dict)) + # Introspection routines (do not advance state, no scheme calls). + # With --no-host-introspection, each routine retains its signature + # but the body is replaced with an errflg=1 stub (or, for + # suite_list, an error_unit write + empty allocation), shrinking + # _ccpp_cap.F90 dramatically for multi-suite builds. + lines.extend(_suite_list_subroutine( + suite_names, stub_body=no_host_introspection, + )) + lines.extend(_suite_part_list_subroutine( + suite_names, suite_resolutions, stub_body=no_host_introspection, + )) + lines.extend(_suite_schemes_subroutine( + suite_names, suite_resolutions, stub_body=no_host_introspection, + )) + lines.extend(_suite_io_subroutine( + suite_names, suite_resolutions, host_dict, collapse_ddts=False, + stub_body=no_host_introspection, + )) + lines.extend(_suite_io_subroutine( + suite_names, suite_resolutions, host_dict, collapse_ddts=True, + stub_body=no_host_introspection, + )) + + lines.append('') + lines.append('end module {}'.format(mod_name)) + return lines + + +######################################################################## +# Public API +######################################################################## + +def write_host_cap( + host_name: str, + suite_names: List[str], + suite_resolutions: List[SuiteResolution], + output_root: str, + host_dict=None, + scheme_store: Optional[SchemeStore] = None, + logger: Optional[logging.Logger] = None, + no_host_introspection: bool = False, + trace: bool = False, +) -> str: + """Write ``_ccpp_cap.F90`` to *output_root*. + + Parameters + ---------- + host_name : str + Host identifier; drives both the file name + (``_ccpp_cap.F90``) and the emitted module name + (``module _ccpp_cap``). + suite_names : list of str + suite_resolutions : list of SuiteResolution + Parallel to suite_names. + output_root : str + Output directory (created if absent). + host_dict : dict, optional + Flat host+control dictionary for multi-instance support. + scheme_store : SchemeStore, optional + Used to detect which suites have at least one ``register``-providing + scheme; only those drive emission of ``ccpp_register`` and the + constituent module USE. When omitted, ``ccpp_register`` is omitted + entirely. + no_host_introspection : bool, optional + When True, replace the bodies of the five suite-introspection + routines (``ccpp_physics_suite_list`` / ``..._suite_part_list`` / + ``..._suite_schemes`` / ``..._suite_variables`` / + ``..._suite_host_data``) with stubs that set ``errflg=1`` and a + clear ``errmsg`` (or write to ``error_unit`` for + ``ccpp_physics_suite_list``, which has no error channel). + Signatures remain so existing callers still link. Use this to + shrink ``_ccpp_cap.F90`` from ~33k lines to ~800 for + multi-suite builds where the introspection case-blocks make + even ``-O1`` compilation impractical. + + Returns + ------- + str + Absolute path of the written file. + """ + os.makedirs(output_root, exist_ok=True) + filename = '{}_ccpp_cap.F90'.format(host_name) + out_path = os.path.join(output_root, filename) + + lines = _generate_host_cap( + host_name, suite_names, suite_resolutions, host_dict, scheme_store, + no_host_introspection=no_host_introspection, + trace=trace, + ) + with open_if_changed(out_path, logger=logger) as fh: + fh.write('\n'.join(lines) + '\n') + return out_path diff --git a/capgen/generator/host_constituents.py b/capgen/generator/host_constituents.py new file mode 100644 index 00000000..6109bb10 --- /dev/null +++ b/capgen/generator/host_constituents.py @@ -0,0 +1,711 @@ +#!/usr/bin/env python3 + +"""Generate ``ccpp_host_constituents.F90`` — the host-wide constituent module. + +In capgen's per-instance design, the constituent state is sized to +``number_of_instances`` (declared by the host's ``type=control`` table, +paired with ``instance_number``). +This module owns: + +* ``ccpp_model_constituents_obj(:)`` — one DDT instance per host + instance, lazily allocated on first ``ccpp_register_constituents``. +* Per-suite buffers ``_dynamic_constituents(:)`` — populated by + ``_register`` from register-phase scheme call results. + Single-dim because registration is identical across instances; the + first instance fills the buffer and subsequent instances skip. +* ``index_of_`` integers (module-level scalars; identical across + instances) — bound by ``ccpp_initialize_constituents`` via + ``%const_index`` queries. +* ``ccpp_model_const_stdnames`` parameter array listing the std names + known to capgen at code-generation time. + +Host-facing API mirrors original capgen but every routine that touches +a specific instance's state takes ``instance_number`` (when the host +declares it). Scheme call sites generated by the resolver access +``ccpp_model_constituents_obj()%vars_layer(…)`` directly, so the +old module-level pointer bindings (``ccpp_constituents`` etc.) have +been removed. +""" + +import logging +import os +from typing import List, Optional, Set, Tuple + +from metadata.parse_tools import open_if_changed +from generator.suite_resolver import SuiteResolution, _index_symbol_name + +_INDENT = ' ' + +_HOST_CONST_MOD = 'ccpp_host_constituents' +_CONST_PROP_MOD = 'ccpp_constituent_prop_mod' +_CONST_DDT = 'ccpp_model_constituents_t' +_CONST_PROP_TYPE = 'ccpp_constituent_properties_t' +_CONST_PROP_PTR_TYPE = 'ccpp_constituent_prop_ptr_t' + +# Public name of the host-wide constituent object (now a per-instance +# allocatable array). +_CONST_OBJ = 'ccpp_model_constituents_obj' + + +######################################################################## +# Aggregation helpers +######################################################################## + +def _any_constituent_state(suite_results: List[SuiteResolution]) -> bool: + """Return True iff any suite either uses or registers constituents.""" + return any( + suite_resolution.uses_constituents or suite_resolution.constituent_register_calls + for suite_resolution in suite_results + ) + + +def _all_index_names(suite_results: List[SuiteResolution]) -> List[str]: + """Return sorted unique base std-names that need an ``index_of_``.""" + names: Set[str] = set() + for suite_resolution in suite_results: + names.update(suite_resolution.constituent_index_names) + return sorted(names) + + +def _suites_with_register_consts( + suite_results: List[SuiteResolution], +) -> List[str]: + # auto-clone-constituents: predicate is centralised on + # SuiteResolution so this module never reads legacy-shim state + # directly; see ``SuiteResolution.needs_dynamic_constituents_buffer``. + return [sr.suite_name for sr in suite_results + if sr.needs_dynamic_constituents_buffer] + + +def _dyn_const_array_name(suite_name: str) -> str: + return '{}_dynamic_constituents'.format(suite_name) + + +def _host_lookup(host_dict, std_name: str) -> Tuple[Optional[str], Optional[str]]: + """Return (local_name, module_name) for *std_name* in *host_dict*. + + Returns ``(None, None)`` when the entry is absent or is a control var. + """ + if not host_dict: + return None, None + entry = host_dict.get(std_name) + if entry is None: + return None, None + return entry.local_name, entry.module_name + + +######################################################################## +# Per-routine emitters +######################################################################## + +def _instance_signature( + base_args: List[str], + inst_local: Optional[str], + ninst_local: Optional[str] = None, +) -> List[str]: + """Return signature args with *inst_local* and (optionally) + *ninst_local* inserted before err args.""" + sig = list(base_args) + if inst_local: + sig.append(inst_local) + if ninst_local: + sig.append(ninst_local) + sig += ['errcode', 'errmsg'] + return sig + + +def _register_constituents_lines( + suite_results: List[SuiteResolution], + host_dict, +) -> List[str]: + """Emit ``ccpp_register_constituents``.""" + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + register_suites = _suites_with_register_consts(suite_results) + inst_local, _ = _host_lookup(host_dict, 'instance_number') + # ``number_of_instances`` is now a paired control variable; it arrives + # as a dummy argument rather than via ``use ``. Its module + # is ``None`` (control vars carry no module), so ninst_mod is ignored. + ninst_local, _ = _host_lookup(host_dict, 'number_of_instances') + inst_idx = inst_local if inst_local else '1' + ninst_arg = ninst_local if ninst_local else '1' + + sig = _instance_signature( + ['host_constituents'], inst_local, ninst_local=ninst_local, + ) + lines: List[str] = [''] + lines.append('{}subroutine ccpp_register_constituents({})'.format( + i1, ', '.join(sig), + )) + lines.append('') + lines.append( + '{}type({}), target, intent(in) :: host_constituents(:)'.format( + i2, _CONST_PROP_TYPE, + ) + ) + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + if ninst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, ninst_local)) + lines.append('{}integer, intent(out) :: errcode'.format(i2)) + lines.append('{}character(len=*), intent(out) :: errmsg'.format(i2)) + lines.append('') + lines.append('{}integer :: num_consts, index'.format(i2)) + lines.append('{}type({}), pointer :: const_prop => null()'.format( + i2, _CONST_PROP_TYPE, + )) + lines.append('') + lines.append("{}errmsg = ''".format(i2)) + lines.append('{}errcode = 0'.format(i2)) + lines.append('') + # Allocate the object array on first call (idempotent across instances). + lines.append('{}if (.not. allocated({})) then'.format(i2, _CONST_OBJ)) + lines.append('{}allocate({}({}))'.format(i3, _CONST_OBJ, ninst_arg)) + lines.append('{}end if'.format(i2)) + lines.append('') + # Count. + lines.append('{}num_consts = size(host_constituents, 1)'.format(i2)) + for sname in register_suites: + buf = _dyn_const_array_name(sname) + lines.append('{}if (allocated({})) then'.format(i2, buf)) + lines.append( + '{}if (allocated({}({})%items)) then'.format(i3, buf, inst_idx) + ) + lines.append( + '{}num_consts = num_consts + size({}({})%items, 1)'.format( + i3 + _INDENT, buf, inst_idx, + ) + ) + lines.append('{}end if'.format(i3)) + lines.append('{}end if'.format(i2)) + lines.append('') + lines.append('{}call {}({})%initialize_table(num_consts)'.format( + i2, _CONST_OBJ, inst_idx, + )) + lines.append('') + # Host constituents first. + lines.append('{}do index = 1, size(host_constituents, 1)'.format(i2)) + lines.append('{}const_prop => host_constituents(index)'.format(i3)) + lines.append( + '{}call {}({})%new_field(const_prop, errcode=errcode, errmsg=errmsg)'.format( + i3, _CONST_OBJ, inst_idx, + ) + ) + lines.append('{}nullify(const_prop)'.format(i3)) + lines.append('{}if (errcode /= 0) return'.format(i3)) + lines.append('{}end do'.format(i2)) + for sname in register_suites: + buf = _dyn_const_array_name(sname) + lines.append('') + lines.append("{}! Merge {} dynamic constituents".format(i2, sname)) + lines.append('{}if (allocated({})) then'.format(i2, buf)) + lines.append( + '{}if (allocated({}({})%items)) then'.format(i3, buf, inst_idx) + ) + lines.append( + '{}do index = 1, size({}({})%items, 1)'.format( + i3 + _INDENT, buf, inst_idx, + ) + ) + lines.append( + '{}const_prop => {}({})%items(index)'.format( + i3 + _INDENT * 2, buf, inst_idx, + ) + ) + lines.append( + '{}call {}({})%new_field(const_prop, errcode=errcode, errmsg=errmsg)'.format( + i3 + _INDENT * 2, _CONST_OBJ, inst_idx, + ) + ) + lines.append('{}nullify(const_prop)'.format(i3 + _INDENT * 2)) + lines.append('{}if (errcode /= 0) return'.format(i3 + _INDENT * 2)) + lines.append('{}end do'.format(i3 + _INDENT)) + lines.append('{}end if'.format(i3)) + lines.append('{}end if'.format(i2)) + lines.append('') + lines.append( + '{}call {}({})%lock_table(errcode=errcode, errmsg=errmsg)'.format( + i2, _CONST_OBJ, inst_idx, + ) + ) + lines.append('') + lines.append('{}end subroutine ccpp_register_constituents'.format(i1)) + return lines + + +def _initialize_constituents_lines( + suite_results: List[SuiteResolution], + host_dict, +) -> List[str]: + """Emit ``ccpp_initialize_constituents``.""" + i1 = _INDENT + i2 = _INDENT * 2 + inst_local, _ = _host_lookup(host_dict, 'instance_number') + inst_idx = inst_local if inst_local else '1' + index_names = _all_index_names(suite_results) + + sig = _instance_signature(['ncols', 'num_layers'], inst_local) + lines: List[str] = [''] + lines.append('{}subroutine ccpp_initialize_constituents({})'.format( + i1, ', '.join(sig), + )) + lines.append( + '{}use ccpp_scheme_utils, only: ccpp_initialize_constituent_ptr'.format(i2) + ) + lines.append('') + lines.append('{}integer, intent(in) :: ncols, num_layers'.format(i2)) + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + lines.append('{}integer, intent(out) :: errcode'.format(i2)) + lines.append('{}character(len=*), intent(out) :: errmsg'.format(i2)) + lines.append('') + lines.append('{}type({}), pointer :: const_obj_ptr => null()'.format( + i2, _CONST_DDT, + )) + lines.append('') + lines.append("{}errmsg = ''".format(i2)) + lines.append('{}errcode = 0'.format(i2)) + lines.append('') + lines.append( + '{}call {}({})%lock_data(ncols, num_layers, errcode=errcode, errmsg=errmsg)'.format( + i2, _CONST_OBJ, inst_idx, + ) + ) + lines.append('{}if (errcode /= 0) return'.format(i2)) + # Cache the singleton pointer in ccpp_scheme_utils (cam-sima compat). + # Only the FIRST call across instances actually sets it (the routine + # is guarded internally); other instances see the first instance's + # object when they call ccpp_constituent_index — a known limitation + # of the framework's scheme_utils for multi-instance hosts. + lines.append('{}const_obj_ptr => {}({})'.format( + i2, _CONST_OBJ, inst_idx, + )) + lines.append('{}call ccpp_initialize_constituent_ptr(const_obj_ptr)'.format(i2)) + lines.append('{}nullify(const_obj_ptr)'.format(i2)) + lines.append('') + for std_name in index_names: + index_sym = _index_symbol_name(std_name) + lines.append( + "{}call {}({})%const_index({}, '{}', " + "errcode=errcode, errmsg=errmsg)".format( + i2, _CONST_OBJ, inst_idx, index_sym, std_name, + ) + ) + lines.append('{}if (errcode /= 0) return'.format(i2)) + # %const_index doesn't error on a miss — it sets the integer to + # int_unassigned and leaves errcode unchanged. Surface that case + # explicitly so the host sees the bad registration at init time + # instead of crashing on a -huge(1) subscript later. + lines.append( + '{}if ({} == int_unassigned) then'.format( + i2, index_sym, + ) + ) + lines.append('{}errcode = 1'.format(i2 + _INDENT)) + lines.append( + "{}errmsg = 'ccpp_initialize_constituents: constituent " + "''{}'' is referenced by a scheme but is not in the " + "registered constituent table; check that some scheme'" + "'s _register routine or the host_constituents argument " + "to ccpp_register_constituents includes it'".format( + i2 + _INDENT, std_name, + ) + ) + lines.append('{}return'.format(i2 + _INDENT)) + lines.append('{}end if'.format(i2)) + lines.append('') + lines.append('{}end subroutine ccpp_initialize_constituents'.format(i1)) + return lines + + +def _is_scheme_constituent_lines(suite_results: List[SuiteResolution]) -> List[str]: + """Emit ``ccpp_is_scheme_constituent`` (no instance_number — module-level).""" + i1 = _INDENT + i2 = _INDENT * 2 + index_names = _all_index_names(suite_results) + lines: List[str] = [''] + lines.append( + '{}subroutine ccpp_is_scheme_constituent(var_name, ' + 'constituent_exists, errcode, errmsg)'.format(i1) + ) + lines.append('') + lines.append('{}character(len=*), intent(in) :: var_name'.format(i2)) + lines.append('{}logical, intent(out) :: constituent_exists'.format(i2)) + lines.append('{}integer, intent(out) :: errcode'.format(i2)) + lines.append('{}character(len=*), intent(out) :: errmsg'.format(i2)) + lines.append('') + lines.append("{}errmsg = ''".format(i2)) + lines.append('{}errcode = 0'.format(i2)) + lines.append('') + if index_names: + lines.append( + '{}constituent_exists = any(ccpp_model_const_stdnames == var_name)'.format(i2) + ) + else: + lines.append('{}constituent_exists = .false.'.format(i2)) + lines.append('') + lines.append('{}end subroutine ccpp_is_scheme_constituent'.format(i1)) + return lines + + +def _wrap_method_sub( + sub_name: str, + method: str, + extra_args: List[Tuple[str, str, str]], + inst_local: Optional[str], + errcode_call: bool = True, +) -> List[str]: + """Thin wrapper subroutine calling ``obj(inst_num)%method(...)``.""" + i1 = _INDENT + i2 = _INDENT * 2 + inst_idx = inst_local if inst_local else '1' + sig = [n for n, _, _ in extra_args] + if inst_local: + sig.append(inst_local) + sig += ['errcode', 'errmsg'] + lines: List[str] = [''] + lines.append('{}subroutine {}({})'.format(i1, sub_name, ', '.join(sig))) + lines.append('') + for _, decl, _ in extra_args: + lines.append('{}{}'.format(i2, decl)) + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + lines.append('{}integer, intent(out) :: errcode'.format(i2)) + lines.append('{}character(len=*), intent(out) :: errmsg'.format(i2)) + lines.append('') + lines.append("{}errmsg = ''".format(i2)) + lines.append('{}errcode = 0'.format(i2)) + lines.append('') + call_args = [call for _, _, call in extra_args] + if errcode_call: + call_args += ['errcode=errcode', 'errmsg=errmsg'] + lines.append('{}call {}({})%{}({})'.format( + i2, _CONST_OBJ, inst_idx, method, ', '.join(call_args), + )) + lines.append('') + lines.append('{}end subroutine {}'.format(i1, sub_name)) + return lines + + +def _wrap_method_subs(host_dict) -> List[str]: + inst_local, _ = _host_lookup(host_dict, 'instance_number') + lines: List[str] = [] + lines.extend(_wrap_method_sub( + 'ccpp_number_constituents', 'num_constituents', + [ + ('num_flds', 'integer, intent(out) :: num_flds', 'num_flds'), + ('advected', 'logical, optional, intent(in) :: advected', + 'advected=advected'), + ], + inst_local, + )) + lines.extend(_wrap_method_sub( + 'ccpp_gather_constituents', 'copy_in', + [('const_array', + 'real(kind=kind_phys), intent(out) :: const_array(:,:,:)', + 'const_array')], + inst_local, + )) + lines.extend(_wrap_method_sub( + 'ccpp_update_constituents', 'copy_out', + [('const_array', + 'real(kind=kind_phys), intent(in) :: const_array(:,:,:)', + 'const_array')], + inst_local, + )) + lines.extend(_wrap_method_sub( + 'ccpp_const_get_index', 'const_index', + [ + ('stdname', 'character(len=*), intent(in) :: stdname', + 'standard_name=stdname'), + ('const_index', 'integer, intent(out) :: const_index', + 'index=const_index'), + ], + inst_local, + )) + return lines + + +def _accessor_functions(host_dict) -> List[str]: + """Pointer-returning accessor functions, indexed by instance_number.""" + i1 = _INDENT + i2 = _INDENT * 2 + inst_local, _ = _host_lookup(host_dict, 'instance_number') + inst_idx = inst_local if inst_local else '1' + args = '({})'.format(inst_local) if inst_local else '()' + decl = ('integer, intent(in) :: {}'.format(inst_local) + if inst_local else '') + + def _emit(fname, method, ret_decl): + out = [ + '', + '{}function {}{} result(const_ptr)'.format(i1, fname, args), + '{}{}'.format(i2, ret_decl), + ] + if decl: + out.append('{}{}'.format(i2, decl)) + out += [ + '{}const_ptr => {}({})%{}()'.format( + i2, _CONST_OBJ, inst_idx, method, + ), + '{}end function {}'.format(i1, fname), + ] + return out + + lines: List[str] = [] + lines += _emit( + 'ccpp_constituents_array', 'field_data_ptr', + 'real(kind=kind_phys), pointer :: const_ptr(:,:,:)', + ) + lines += _emit( + 'ccpp_advected_constituents_array', 'advected_constituents_ptr', + 'real(kind=kind_phys), pointer :: const_ptr(:,:,:)', + ) + # Properties function returns a different pointer type. + fname = 'ccpp_model_const_properties' + ret_decl = 'type({}), pointer :: const_ptr(:)'.format(_CONST_PROP_PTR_TYPE) + args_props = '({})'.format(inst_local) if inst_local else '()' + lines += [ + '', + '{}function {}{} result(const_ptr)'.format(i1, fname, args_props), + '{}{}'.format(i2, ret_decl), + ] + if decl: + lines.append('{}{}'.format(i2, decl)) + lines += [ + '{}const_ptr => {}({})%constituent_props_ptr()'.format( + i2, _CONST_OBJ, inst_idx, + ), + '{}end function {}'.format(i1, fname), + ] + return lines + + +def _deallocate_lines( + suite_results: List[SuiteResolution], + host_dict, +) -> List[str]: + """Emit ``ccpp_deallocate_dynamic_constituents`` — per-instance with + last-to-leave teardown. + + Each call resets the constituent object for the given instance. When + every instance's object has been reset (none still has its property + table locked), the per-suite dynamic-constituent buffers and the + object array itself are deallocated, and the cached + ``index_of_`` integers are reset. + + Mirrors the per-instance + last-to-leave pattern used by + ``_final`` in the suite cap. + """ + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + register_suites = _suites_with_register_consts(suite_results) + index_names = _all_index_names(suite_results) + inst_local, _ = _host_lookup(host_dict, 'instance_number') + inst_idx = inst_local if inst_local else '1' + + sig_args = [inst_local] if inst_local else [] + + lines: List[str] = [''] + lines.append('{}subroutine ccpp_deallocate_dynamic_constituents({})'.format( + i1, ', '.join(sig_args), + )) + lines.append('') + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + lines.append('') + lines.append('{}integer :: i'.format(i2)) + lines.append('{}logical :: all_done'.format(i2)) + lines.append('') + lines.append('{}if (.not. allocated({})) return'.format(i2, _CONST_OBJ)) + lines.append('') + # Per-instance reset. + lines.append('{}call {}({})%reset()'.format(i2, _CONST_OBJ, inst_idx)) + lines.append('') + # Last-to-leave: when no instance still has its constituent table + # locked, tear down everything (buffers + object array + indices). + lines.append('{}all_done = .true.'.format(i2)) + lines.append('{}do i = 1, size({}, 1)'.format(i2, _CONST_OBJ)) + lines.append('{}if ({}(i)%const_props_locked()) then'.format( + i3, _CONST_OBJ, + )) + lines.append('{}all_done = .false.'.format(i3 + _INDENT)) + lines.append('{}exit'.format(i3 + _INDENT)) + lines.append('{}end if'.format(i3)) + lines.append('{}end do'.format(i2)) + lines.append('') + lines.append('{}if (all_done) then'.format(i2)) + # The per-suite ``_dynamic_constituents`` buffers are NOT + # deallocated here. They're owned by the suite-cap lifecycle (filled + # by ``_register``, gated by the suite-cap state machine), so + # tearing them down independently would leave the suite_state in a + # state where the next ``ccpp_register`` short-circuits via the + # ``state >= REGISTERED`` guard and the buffer never gets re-filled. + # They are deallocated in ``_final``'s last-to-leave block + # instead. + lines.append('{}deallocate({})'.format(i3, _CONST_OBJ)) + # Reset to the unbound sentinel (matches the declaration default) so a + # re-register / re-init cycle starts from int_unassigned and the guard + # stays effective on the second pass. + for std_name in index_names: + lines.append('{}{} = int_unassigned'.format( + i3, _index_symbol_name(std_name))) + lines.append('{}end if'.format(i2)) + lines.append('') + lines.append('{}end subroutine ccpp_deallocate_dynamic_constituents'.format(i1)) + return lines + + +######################################################################## +# Top-level generator +######################################################################## + +def _generate_host_constituents( + suite_results: List[SuiteResolution], + host_dict=None, +) -> Optional[List[str]]: + """Generate ``ccpp_host_constituents.F90`` source lines, or ``None``.""" + if not _any_constituent_state(suite_results): + return None + + register_suites = _suites_with_register_consts(suite_results) + index_names = _all_index_names(suite_results) + + lines: List[str] = [] + lines.append( + '! ccpp_host_constituents.F90 -- generated by ccpp_capgen, do not edit' + ) + lines.append('module {}'.format(_HOST_CONST_MOD)) + lines.append('') + lines.append('{}use ccpp_kinds, only: kind_phys'.format(_INDENT)) + lines.append('{}use {}, only: &'.format(_INDENT, _CONST_PROP_MOD)) + lines.append('{}{}, &'.format(_INDENT * 2, _CONST_DDT)) + lines.append('{}{}, &'.format(_INDENT * 2, _CONST_PROP_TYPE)) + lines.append('{}{}, &'.format(_INDENT * 2, _CONST_PROP_PTR_TYPE)) + # int_unassigned is the sentinel returned by %const_index when a + # standard name is not in the constituent table. Used by + # ccpp_initialize_constituents to validate that every cached + # index_of_ integer corresponds to an actually-registered + # constituent — surfaces missing-registration bugs at init time + # instead of letting them become silent invalid array accesses + # later in run-phase scheme calls. + lines.append('{}int_unassigned'.format(_INDENT * 2)) + lines.append('') + lines.append('{}implicit none'.format(_INDENT)) + lines.append('{}private'.format(_INDENT)) + lines.append('') + + # Publics: state + routines. + publics = [_CONST_OBJ] + publics += [_index_symbol_name(n) for n in index_names] + publics += [ + 'ccpp_register_constituents', + 'ccpp_initialize_constituents', + 'ccpp_is_scheme_constituent', + 'ccpp_number_constituents', + 'ccpp_gather_constituents', + 'ccpp_update_constituents', + 'ccpp_const_get_index', + 'ccpp_constituents_array', + 'ccpp_advected_constituents_array', + 'ccpp_model_const_properties', + 'ccpp_deallocate_dynamic_constituents', + ] + if index_names: + publics.append('ccpp_model_const_stdnames') + publics += [_dyn_const_array_name(s) for s in register_suites] + for p in publics: + lines.append('{}public :: {}'.format(_INDENT, p)) + lines.append('') + + # Per-instance wrapper around the per-suite scheme-registered + # constituent buffer. Each instance owns its own slot in the outer + # array; the inner ``items(:)`` array is filled independently by + # ``_register`` per instance so that the property objects + # (which acquire a const_ind during ``ccpp_register_constituents``) + # are not shared across instances. + if register_suites: + lines.append( + '{}type :: ccpp_dyn_const_buffer_t'.format(_INDENT) + ) + lines.append( + '{}type({}), allocatable :: items(:)'.format( + _INDENT * 2, _CONST_PROP_TYPE, + ) + ) + lines.append( + '{}end type ccpp_dyn_const_buffer_t'.format(_INDENT) + ) + lines.append('') + + # State declarations. + lines.append( + '{}type({}), target, allocatable :: {}(:)'.format( + _INDENT, _CONST_DDT, _CONST_OBJ, + ) + ) + lines.append('') + for sname in register_suites: + buf = _dyn_const_array_name(sname) + lines.append( + '{}type(ccpp_dyn_const_buffer_t), allocatable, target :: {}(:)'.format( + _INDENT, buf, + ) + ) + if register_suites: + lines.append('') + # Default to int_unassigned (NOT 0) so a constituent that is referenced + # by a scheme but never bound by ccpp_initialize_constituents (missing + # registration, or a host that never calls the init routine) is caught + # by the post-const_index guard below instead of silently surviving as + # an index of 0 and producing an out-of-bounds vars_layer subscript at + # run time. This also keeps the guard correct for any framework whose + # %const_index leaves the output unchanged on a name miss. + for std_name in index_names: + lines.append('{}integer :: {} = int_unassigned'.format( + _INDENT, _index_symbol_name(std_name))) + if index_names: + max_len = max(len(n) for n in index_names) + lines.append('') + lines.append( + '{}character(len={}), parameter :: ccpp_model_const_stdnames({}) = (/ &'.format( + _INDENT, max_len, len(index_names), + ) + ) + for i, std_name in enumerate(index_names): + sep = ', &' if i < len(index_names) - 1 else ' /)' + padding = ' ' * (max_len - len(std_name)) + lines.append("{}'{}{}'{}".format(_INDENT * 2, std_name, padding, sep)) + lines.append('') + lines.append('contains') + + lines.extend(_register_constituents_lines(suite_results, host_dict)) + lines.extend(_initialize_constituents_lines(suite_results, host_dict)) + lines.extend(_is_scheme_constituent_lines(suite_results)) + lines.extend(_wrap_method_subs(host_dict)) + lines.extend(_accessor_functions(host_dict)) + lines.extend(_deallocate_lines(suite_results, host_dict)) + + lines.append('') + lines.append('end module {}'.format(_HOST_CONST_MOD)) + return lines + + +def write_host_constituents( + suite_results: List[SuiteResolution], + outdir: str, + host_dict=None, + logger: Optional[logging.Logger] = None, +) -> Optional[str]: + """Write ``ccpp_host_constituents.F90`` if needed, return its path or ``None``.""" + lines = _generate_host_constituents(suite_results, host_dict) + if lines is None: + return None + if not os.path.isdir(outdir): + os.makedirs(outdir, exist_ok=True) + path = os.path.join(outdir, 'ccpp_host_constituents.F90') + with open_if_changed(path, logger=logger) as fh: + fh.write('\n'.join(lines) + '\n') + return os.path.abspath(path) diff --git a/capgen/generator/kinds_writer.py b/capgen/generator/kinds_writer.py new file mode 100644 index 00000000..c6cbf4dd --- /dev/null +++ b/capgen/generator/kinds_writer.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 + +"""Write the ``ccpp_kinds.F90`` module from ``--kind-type`` mappings. + +This is the simplest generated file -- a single Fortran module that re-exports +host-supplied kind parameters as ``integer, parameter, public`` constants. + +Each kind is described by a ``(module, spec)`` pair: the Fortran module that +defines the precision constant (``module``) and the name of that constant +(``spec``). When the spec is a standard ``ISO_FORTRAN_ENV`` name (e.g. +``REAL64``), the user may omit the module on the command line and it defaults +to ``iso_fortran_env``. When the host supplies its own kind module, the +caller passes ``(, )``. + +Example output for ``--kind-type kind_phys=REAL64`` (default module):: + + ! ccpp_kinds.F90 -- generated by ccpp_capgen, do not edit + module ccpp_kinds + use iso_fortran_env, only: REAL64 + + implicit none + private + + integer, parameter, public :: kind_phys = REAL64 + + end module ccpp_kinds + +Example output for ``--kind-type kind_phys=my_host_kinds:kind_r8`` (host module):: + + ! ccpp_kinds.F90 -- generated by ccpp_capgen, do not edit + module ccpp_kinds + use my_host_kinds, only: kind_r8 + + implicit none + private + + integer, parameter, public :: kind_phys = kind_r8 + + end module ccpp_kinds + +``ccpp_kinds.F90`` is always generated and is a dependency of all other +generated Fortran files that reference any kind parameter. +""" + +import logging +import os +from typing import Dict, List, Optional, Tuple + +from metadata.parse_tools import CCPPError, open_if_changed + +_KINDS_FILENAME = 'ccpp_kinds.F90' +_KINDS_MODULE = 'ccpp_kinds' + +# Mapping from kind name to a (module, spec) pair. +KindSpec = Tuple[str, str] +KindMap = Dict[str, KindSpec] + + +######################################################################## +# Public API +######################################################################## + +def write_ccpp_kinds( + kind_types: KindMap, + output_root: str, + logger: Optional[logging.Logger] = None, +) -> str: + """Write ``ccpp_kinds.F90`` to *output_root*. + + Parameters + ---------- + kind_types : dict + Mapping ``kind_name -> (module_name, kind_spec)``. Example:: + + {'kind_phys': ('iso_fortran_env', 'REAL64'), + 'kind_dyn': ('my_host_kinds', 'kind_r4')} + + output_root : str + Directory where the file is written (created if absent). + + Returns + ------- + str + Absolute path of the written file. + + Raises + ------ + CCPPError + If *kind_types* is empty. + """ + os.makedirs(output_root, exist_ok=True) + lines = _generate_ccpp_kinds(kind_types) + out_path = os.path.join(output_root, _KINDS_FILENAME) + with open_if_changed(out_path, logger=logger) as fh: + fh.write('\n'.join(lines) + '\n') + return out_path + + +######################################################################## +# Internal generator +######################################################################## + +def _generate_ccpp_kinds(kind_types: KindMap) -> List[str]: + """Generate the ccpp_kinds module source as a list of lines. + + Lines do not carry trailing newlines. The caller joins them and + appends a final newline. + + Parameters + ---------- + kind_types : dict + Mapping ``kind_name -> (module_name, kind_spec)``. Keys are sorted + alphabetically in the output; ``use`` lines are grouped by module + and sorted alphabetically by module name. + + Returns + ------- + list of str + + Raises + ------ + CCPPError + If *kind_types* is empty. + + Examples + -------- + >>> lines = _generate_ccpp_kinds({'kind_phys': ('iso_fortran_env', 'REAL64')}) + >>> lines[0] + '! ccpp_kinds.F90 -- generated by ccpp_capgen, do not edit' + >>> 'module ccpp_kinds' in lines + True + >>> any('kind_phys' in l and 'REAL64' in l for l in lines) + True + >>> 'end module ccpp_kinds' in lines + True + + Two distinct kinds, both from ``iso_fortran_env``: + + >>> lines = _generate_ccpp_kinds({ + ... 'kind_phys': ('iso_fortran_env', 'REAL64'), + ... 'kind_dyn': ('iso_fortran_env', 'REAL32'), + ... }) + >>> sum(1 for l in lines if 'parameter' in l and 'public' in l) + 2 + >>> sum(1 for l in lines if 'use iso_fortran_env' in l) + 1 + >>> any('REAL32' in l and 'REAL64' in l for l in lines) + True + + Two kinds sharing the same spec name -- the spec is listed once: + + >>> lines = _generate_ccpp_kinds({ + ... 'a': ('iso_fortran_env', 'REAL64'), + ... 'b': ('iso_fortran_env', 'REAL64'), + ... }) + >>> sum(1 for l in lines if 'REAL64' in l and 'iso_fortran_env' in l) + 1 + + Host-supplied module: + + >>> lines = _generate_ccpp_kinds({ + ... 'kind_phys': ('my_host_kinds', 'kind_r8'), + ... }) + >>> any('use my_host_kinds, only: kind_r8' in l for l in lines) + True + >>> any('kind_phys = kind_r8' in l for l in lines) + True + + Empty mapping is an error: + + >>> _generate_ccpp_kinds({}) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + metadata.parse_tools.parse_source.CCPPError: ccpp_kinds requires at least ... + """ + if not kind_types: + raise CCPPError( + "ccpp_kinds requires at least one kind mapping; " + "this is a generator bug -- the caller should inject " + "kind_phys=iso_fortran_env:REAL64 when no --kind-type is given" + ) + + # Group specs by module: {module: sorted_unique_specs} + by_module: Dict[str, List[str]] = {} + for _, (mod, spec) in kind_types.items(): + by_module.setdefault(mod, []) + if spec not in by_module[mod]: + by_module[mod].append(spec) + for mod in by_module: + by_module[mod].sort() + + lines: List[str] = [ + '! ccpp_kinds.F90 -- generated by ccpp_capgen, do not edit', + 'module {}'.format(_KINDS_MODULE), + ] + + # ``use`` statements -- one per module, alphabetized; specs comma-separated. + sorted_modules = sorted(by_module) + use_lines: List[str] = [] + for mod in sorted_modules: + specs = by_module[mod] + use_lines.append(' use {}, only: {}'.format(mod, ', '.join(specs))) + # Column-align the ``only:`` clauses for readability when there is more + # than one ``use`` line. + if len(use_lines) > 1: + max_mod_len = max(len(mod) for mod in sorted_modules) + use_lines = [] + for mod in sorted_modules: + pad = ' ' * (max_mod_len - len(mod)) + specs = by_module[mod] + use_lines.append( + ' use {},{} only: {}'.format(mod, pad, ', '.join(specs)) + ) + lines.extend(use_lines) + + lines += [ + '', + ' implicit none', + ' private', + '', + ] + + # Parameter declarations, column-aligned on '='. + sorted_kinds = sorted(kind_types.items()) + max_len = max(len(k) for k, _ in sorted_kinds) + for kind_name, (_mod, spec) in sorted_kinds: + pad = ' ' * (max_len - len(kind_name)) + lines.append( + ' integer, parameter, public :: {}{} = {}'.format( + kind_name, pad, spec + ) + ) + + lines += [ + '', + 'end module {}'.format(_KINDS_MODULE), + ] + + return lines diff --git a/capgen/generator/suite_cap.py b/capgen/generator/suite_cap.py new file mode 100644 index 00000000..af3a8ddd --- /dev/null +++ b/capgen/generator/suite_cap.py @@ -0,0 +1,1352 @@ +#!/usr/bin/env python3 + +"""Generate the suite-level cap module ``ccpp__cap.F90``. + +The suite cap: + +* Imports all group cap modules and the constituent property module. +* Exposes eight public entry points: + + - ``_register`` — calls each scheme's ``_register`` to populate + the host-owned ``ccpp_model_constituents_t`` object. + - ``_init`` / ``_final`` — framework setup / teardown. + - ``_physics_init``, ``_physics_timestep_init``, + ``_physics_run``, ``_physics_timestep_final``, + ``_physics_final`` — dispatch by ``group_name`` to the + appropriate group cap subroutine. + +The static API (``_ccpp_cap.F90``) dispatches by ``suite_name`` to +these subroutines. +""" + +import logging +import os +from typing import Dict, List, Optional, Set + +from metadata.parse_tools import open_if_changed +from metadata.variable_resolver import HostVarEntry, SchemeStore +from generator.suite_resolver import ( + ResolvedArg, + ResolvedGroup, + SuiteResolution, + iter_phase_calls, + _root_symbol, + # auto-clone-constituents: legacy-shim payload type. + AutoCloneEntry, +) +from generator.trace import ( + emit_module_gate, + emit_trace_block, + ensure_error_unit_use, +) +from generator.group_cap import ( + _ctrl_args_for_phase, + _ctrl_intent_for, + _ctrl_local, + _extra_dim_ctrl_entries, + _ctrl_entries_for_signature, + _fortran_type_str, + _dim_decl, + _instance_idx, + _instance_local, + _intent_clause, +) + +_INDENT = ' ' + +# Canonical set of physics phases, always dispatched by the suite cap. +_PHYSICS_PHASES = ('init', 'timestep_init', 'run', 'timestep_final', 'final') + +# Constituent type / module constants. +_CONST_MOD = 'ccpp_constituent_prop_mod' +_CONST_DDT = 'ccpp_model_constituents_t' +_CONST_PROP_TYPE = 'ccpp_constituent_properties_t' +_CONST_PROP_PTR_TYPE = 'ccpp_constituent_prop_ptr_t' +_CONST_OBJ_STDNAME = 'ccpp_model_constituents_object' + +# Framework-provided constituent symbol names — emitted as suite-cap +# module variables when the suite references constituent state. +_CONST_BASE_ARRAY = 'ccpp_constituents' +_CONST_TEND_ARRAY = 'ccpp_constituent_tendencies' +_CONST_PROPS = 'ccpp_constituent_properties' +_CONST_NUM = 'number_of_ccpp_constituents' + + +######################################################################## +# Helpers +######################################################################## + +def _all_suite_scheme_names(suite_res: SuiteResolution) -> List[str]: + """Return deduplicated scheme names from all groups and phases. + + The order is first-seen across groups (alphabetical by group, then by + order within each group's phase call list). + + >>> from generator.suite_resolver import SuiteResolution, ResolvedGroup, ResolvedCall + >>> resolved_group = ResolvedGroup('grp', phase_calls={'run': [ResolvedCall('sch_a', 'run'), ResolvedCall('sch_b', 'run')]}) + >>> suite_resolution = SuiteResolution('s', groups=[resolved_group]) + >>> _all_suite_scheme_names(suite_resolution) + ['sch_a', 'sch_b'] + """ + seen: Set[str] = set() + names: List[str] = [] + for resolved_group in suite_res.groups: + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + if resolved_call.scheme_name not in seen: + seen.add(resolved_call.scheme_name) + names.append(resolved_call.scheme_name) + return names + + +def _schemes_with_register( + scheme_names: List[str], + scheme_store: SchemeStore, +) -> List[str]: + """Return those scheme names that have a ``register`` phase. + + >>> from unittest.mock import MagicMock + >>> store = MagicMock() + >>> store.phases_for.side_effect = lambda n: ['register', 'run'] if n == 'sch_a' else ['run'] + >>> _schemes_with_register(['sch_a', 'sch_b'], store) + ['sch_a'] + """ + return [n for n in scheme_names if 'register' in scheme_store.phases_for(n)] + + +def _suite_ctrl_args_for_phase( + suite_res: SuiteResolution, + phase: str, +) -> List[ResolvedArg]: + """Return the union of control args across all groups for *phase*. + + The result is deduplicated by standard_name and preserves first-seen order. + """ + seen: Dict[str, ResolvedArg] = {} + for resolved_group in suite_res.groups: + for arg in _ctrl_args_for_phase(resolved_group, phase): + if arg.standard_name not in seen: + seen[arg.standard_name] = arg + return list(seen.values()) + + +def _group_ctrl_arg_names(resolved_group: ResolvedGroup, phase: str, host_dict=None) -> List[str]: + """Return the local_name list for the control args of a group phase. + + These are the keyword names passed when calling the group cap subroutine. + Includes extra control vars needed for state indexing and dimension subscripts + (instance_number for suite-var access, control vars used only in dim subscripts). + """ + ctrl_args = _ctrl_args_for_phase(resolved_group, phase) + names = [ + a.host_entry.local_name + for a in ctrl_args + if a.host_entry is not None + ] + phase_items = resolved_group.phase_calls.get(phase, []) + for entry in _extra_dim_ctrl_entries(phase_items, phase, ctrl_args, host_dict): + if entry.local_name not in names: + names.append(entry.local_name) + return names + + +def _suite_extra_ctrl_entries_for_phase( + suite_res: SuiteResolution, + phase: str, + ctrl_std_names: Set[str], + host_dict, +) -> List[HostVarEntry]: + """Return extra HostVarEntry objects needed by any group for *phase* but not + already represented in *ctrl_std_names* (the direct scheme control args). + + This covers the same cases as ``_extra_dim_ctrl_entries`` but aggregated + across all groups so the suite-level dispatch subroutine has them in its + signature and can pass them down. + """ + if host_dict is None: + return [] + seen = set(ctrl_std_names) + result: Dict[str, HostVarEntry] = {} + for resolved_group in suite_res.groups: + phase_items = resolved_group.phase_calls.get(phase, []) + ctrl_args = _ctrl_args_for_phase(resolved_group, phase) + for entry in _extra_dim_ctrl_entries(phase_items, phase, ctrl_args, host_dict): + if entry.standard_name not in seen and entry.standard_name not in result: + result[entry.standard_name] = entry + return list(result.values()) + + +######################################################################## +# Subroutine generators +######################################################################## + +def _register_calls(suite_res: SuiteResolution): + """Yield (group_name, ResolvedCall) for every register-phase scheme call. + + Groups are visited in suite-XML order; within each group the calls follow + the resolver's ordering (which mirrors the suite XML). + """ + for resolved_group in suite_res.groups: + for resolved_call in iter_phase_calls(resolved_group.phase_calls.get('register', [])): + yield resolved_group.group_name, resolved_call + + +def _register_uses( + suite_res: SuiteResolution, + suite_name: str, + host_dict=None, +) -> Dict[str, Set[str]]: + """Collect ``{module: {symbol}}`` requirements for register-phase scheme calls. + + Includes: + - host modules for any host-owned register args, + - ``ccpp__data`` for any suite-owned register args, + - one entry per scheme module for its ``_register`` symbol, + - the constituent property type and the per-suite dynamic-constituent + buffer (owned by ``ccpp_host_constituents``) when any register call + produces constituents. + """ + uses: Dict[str, Set[str]] = {} + seen_schemes: Set[str] = set() + for _gname, resolved_call in _register_calls(suite_res): + if resolved_call.scheme_name not in seen_schemes: + seen_schemes.add(resolved_call.scheme_name) + # Module is metadata-declared (``module_name`` in table props) + # when present; otherwise falls back to the scheme name. + scheme_module = resolved_call.scheme_module or resolved_call.scheme_name + uses.setdefault(scheme_module, set()).add( + '{}_register'.format(resolved_call.scheme_name) + ) + for arg in resolved_call.args: + if arg.is_constituent_arg: + continue # local temp, not a USE'd var + mod = arg.module_name + if mod is not None: + uses.setdefault(mod, set()).add(arg.root_symbol) + # Dimension variables referenced in the arg's subscript (e.g. + # ``rad_climate(1:rad_climate_dimension)``) must also be in + # scope. Mirror the group cap's _collect_dim_uses: a host dim + # USEs its access-path root from the declaring module; a + # suite-owned dim USEs ccpp__data. (Without this the + # register subroutine references the dimension symbol with no + # IMPLICIT type.) + for dim_std in arg.used_dim_std_names: + entry = host_dict.get(dim_std) if host_dict else None + if entry is not None and entry.module_name is not None: + uses.setdefault(entry.module_name, set()).add( + _root_symbol(entry.access_path) + ) + elif dim_std in suite_res.suite_vars: + sv = suite_res.suite_vars[dim_std] + uses.setdefault(sv.module_name, set()).add('ccpp_suite_data') + # Per-suite dynamic-constituent buffer is owned by ccpp_host_constituents + # and written into here. Pull in the constituent property type plus the + # buffer symbol. + # auto-clone-constituents: predicate centralised on SuiteResolution so + # this site does not read legacy-shim state directly. + if suite_res.needs_dynamic_constituents_buffer: + uses.setdefault(_CONST_MOD, set()).add(_CONST_PROP_TYPE) + buf = '{}_dynamic_constituents'.format(suite_name) + uses.setdefault('ccpp_host_constituents', set()).add(buf) + # auto-clone-constituents: the synthesised %instantiate calls + # embed ``_kind_phys`` literals when an entry sets any of + # molar_mass / default_value / min_value. Pull in ``kind_phys`` + # from ccpp_kinds in that case so the literal resolves. + if any( + e.default_value is not None + or e.min_value is not None + or (e.molar_mass and e.molar_mass != 0.0) + for e in suite_res.auto_cloned_constituents + ): + uses.setdefault('ccpp_kinds', set()).add('kind_phys') + return uses + + +def _add_call_uses(uses: Dict[str, Set[str]], resolved_call) -> None: + """Merge USE-statement requirements for a single :class:`ResolvedCall`. + + Adds: + * the scheme module → ``_`` symbol so the call + site can resolve; + * each non-control arg's host/suite module → ``arg.root_symbol`` + (the top-level token of its access path) so the value is in + scope. + + Mutates *uses* in place. Used by ``_init`` and + ``_final`` to integrate the suite-level / + scheme calls into the USE block. + """ + scheme_module = resolved_call.scheme_module or resolved_call.scheme_name + uses.setdefault(scheme_module, set()).add( + '{}_{}'.format(resolved_call.scheme_name, resolved_call.phase) + ) + for arg in resolved_call.args: + mod = arg.module_name + if mod is not None: + uses.setdefault(mod, set()).add(arg.root_symbol) + + +def _emit_register_call(resolved_call, indent: str, errflg_local: str, lines: List[str]) -> None: + """Emit one scheme ``_register`` call with keyword args + error guard. + + Register-phase calls are kept simple: no transformations (transform code + paths are physics-phase only), keyword-arg style for clarity. + """ + sub = '{}_register'.format(resolved_call.scheme_name) + if not resolved_call.args: + lines.append('{}call {}()'.format(indent, sub)) + else: + lines.append('{}call {}( &'.format(indent, sub)) + for i, arg in enumerate(resolved_call.args): + sep = ', &' if i < len(resolved_call.args) - 1 else ')' + lines.append('{} {}={}{}'.format( + indent, arg.scheme_local_name, arg.call_expr, sep + )) + if errflg_local: + lines.append('{}if ({} /= 0) return'.format(indent, errflg_local)) + + +# auto-clone-constituents: BEGIN legacy-shim emission helpers. +# Delete this block together with the rest of the +# auto-clone-constituents touchpoints. + +def _esc_fortran_char(value: str) -> str: + """Escape a Python string for embedding in a Fortran character + literal — double every embedded single quote.""" + return value.replace("'", "''") + + +def _fmt_kind_phys_real(value) -> str: + """Format a Python float as a Fortran ``kind_phys`` real literal. + + Always emits exponent notation so the result is unambiguously a + real (no risk of being parsed as an integer literal) and reuses + the same ``kind_phys`` suffix the framework's ``%instantiate`` + declares for ``default_value`` / ``min_value`` / ``molar_mass``. + """ + return '{:.17e}_kind_phys'.format(float(value)) + + +def _emit_auto_clone_instantiate( + entry: AutoCloneEntry, + buf: str, + inst_idx: str, + indent: str, + errflg_local: str, + errmsg_local: str, + lines: List[str], +) -> None: + """auto-clone-constituents: emit one synthesised ``%instantiate`` + call into the per-suite dynamic-constituents buffer for one + :class:`AutoCloneEntry`. + + Required kwargs (std_name, long_name, diag_name, units, + vertical_dim) are always emitted. Optional kwargs are emitted + only when the metadata explicitly set the value (``None`` on the + backing field means "unset" — let the framework default kick in). + ``errcode`` and ``errmsg`` are passed by keyword for clarity. + """ + i = indent + lines.append('{}num_consts = num_consts + 1'.format(i)) + lines.append( + '{}call {}({})%items(num_consts)%instantiate( &'.format( + i, buf, inst_idx, + ) + ) + lines.append("{} std_name = '{}', &".format( + i, _esc_fortran_char(entry.std_name))) + lines.append("{} long_name = '{}', &".format( + i, _esc_fortran_char(entry.long_name))) + lines.append("{} diag_name = '{}', &".format( + i, _esc_fortran_char(entry.diag_name))) + lines.append("{} units = '{}', &".format( + i, _esc_fortran_char(entry.units))) + lines.append("{} vertical_dim = '{}', &".format( + i, _esc_fortran_char(entry.vertical_dim))) + # Optional kwargs — only emit when explicitly set. ``advected`` + # defaults to .false. in metadata; only emit when True so we + # don't pollute the call with a redundant kwarg. + if entry.advected: + lines.append("{} advected = .true., &".format(i)) + if entry.molar_mass and entry.molar_mass != 0.0: + lines.append("{} molar_mass = {}, &".format( + i, _fmt_kind_phys_real(entry.molar_mass))) + if entry.default_value is not None: + lines.append("{} default_value= {}, &".format( + i, _fmt_kind_phys_real(entry.default_value))) + if entry.min_value is not None: + lines.append("{} min_value = {}, &".format( + i, _fmt_kind_phys_real(entry.min_value))) + if entry.water_species is not None: + lines.append("{} water_species= .{}., &".format( + i, 'true' if entry.water_species else 'false')) + if entry.mixing_ratio_type is not None: + lines.append("{} mixing_ratio_type = '{}', &".format( + i, _esc_fortran_char(entry.mixing_ratio_type))) + lines.append("{} errcode = {}, &".format(i, errflg_local)) + lines.append("{} errmsg = {})".format(i, errmsg_local)) + lines.append('{}if ({} /= 0) return'.format(i, errflg_local)) + +# auto-clone-constituents: END legacy-shim emission helpers. + + +def _register_lines( + suite_name: str, + suite_res: SuiteResolution, + host_dict=None, +) -> List[str]: + """Generate the ``_register`` subroutine lines. + + Mandatory entry point: emitted unconditionally. Allocates the suite + state array and the suite-owned DDT array on first call, dispatches each + register-phase scheme call across all groups in suite-XML order, and + transitions the suite state for this instance to ``CCPP_SUITE_REGISTERED``. + + Minimal signature: ``(instance_number, number_of_instances, errmsg, + errflg)`` (the instance pair is included only when the host declares it). + """ + sub_name = '{}_register'.format(suite_name) + i1 = _INDENT + i2 = _INDENT * 2 + + inst_local = _instance_local(host_dict) + inst_idx = _instance_idx(host_dict) + + # ``number_of_instances`` is now a paired control variable (see + # ccpp_capgen._PAIRED_OPTIONAL_CTRL_VARS). When present it enters + # the suite-cap signature as a dummy alongside ``instance_number``; + # the framework consumes it at register-time to size the per-instance + # state arrays. When absent we fall back to the literal ``1`` + # (single-instance API). + ninstances_entry = host_dict.get('number_of_instances') if host_dict else None + ninstances_local = ninstances_entry.local_name if ninstances_entry else None + ninstances_arg = ninstances_local if ninstances_local else '1' + + errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') or 'errflg' + errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') or 'errmsg' + + sig_args: List[str] = [] + if inst_local: + sig_args.append(inst_local) + if ninstances_local: + sig_args.append(ninstances_local) + sig_args += [errmsg_local, errflg_local] + + lines: List[str] = [] + lines.append('') + lines.append('{}subroutine {}({})'.format(i1, sub_name, ', '.join(sig_args))) + + # USE statements: scheme modules + host/suite-data modules referenced by + # register-phase scheme args. (``number_of_instances`` used to be USE'd + # from the host module here; now it arrives as a dummy argument.) + reg_uses = _register_uses(suite_res, suite_name, host_dict) + for mod in sorted(reg_uses): + syms = ', '.join(sorted(reg_uses[mod])) + lines.append('{}use {}, only: {}'.format(i2, mod, syms)) + + lines.append('') + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + if ninstances_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, ninstances_local)) + lines += [ + '{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local), + '{}integer, intent(out) :: {}'.format(i2, errflg_local), + ] + + # Constituent merge: declare a per-scheme array temporary and a counter. + has_consts = bool(suite_res.constituent_register_calls) + # auto-clone-constituents: the legacy shim contributes additional + # constituent registrations synthesised in capgen from + # is_constituent consumer metadata; ``has_dyn_consts`` covers + # both sources so the buffer-allocation + counter machinery is + # set up whenever any synthesised constituent will be emitted. + has_auto_cloned = bool(suite_res.auto_cloned_constituents) + has_dyn_consts = has_consts or has_auto_cloned + if has_dyn_consts: + lines.append('') + if has_consts: + # Each constituent scheme's _register returns its array into this + # temp; it is appended to the buffer and reused for the next + # scheme, so _register is called EXACTLY ONCE per scheme. + lines.append( + '{}type({}), allocatable :: scheme_consts(:)'.format( + i2, _CONST_PROP_TYPE + ) + ) + if has_auto_cloned: + # Counter used only by the auto-clone-constituents buffer growth. + lines.append('{}integer :: num_consts'.format(i2)) + + # Trace block: dummies referenced inside the gated write so strict + # compilers don't flag intent(in) args as unused when the gate is off. + extra_in = [ninstances_local] if ninstances_local else None + trace_lines = emit_trace_block( + sub_name, [], i2, + instance_local=inst_local, extra_in_names=extra_in, + ) + if trace_lines: + lines.append('') + lines.extend(trace_lines) + + lines += [ + '', + "{}{} = ''".format(i2, errmsg_local), + '{}{} = 0'.format(i2, errflg_local), + '', + ] + + # Allocate state and DDT array on first call (idempotent). + suite_alloc_sub = '{}_suite_state_alloc'.format(suite_name) + lines.append('{}call {}({}, {}, {})'.format( + i2, suite_alloc_sub, ninstances_arg, errmsg_local, errflg_local + )) + lines.append('{}if ({} /= 0) return'.format(i2, errflg_local)) + lines.append('') + + # Per-instance idempotent skip: already registered or further along. + lines.append( + '{}if (ccpp_suite_state({}) >= CCPP_SUITE_REGISTERED) return'.format( + i2, inst_idx + ) + ) + lines.append('') + + if has_dyn_consts: + # Pack constituent-producing schemes' arrays into the per-suite + # buffer in ccpp_host_constituents. The actual merge into each + # instance's ``ccpp_model_constituents_obj(inst)`` happens later + # when the host calls ``ccpp_register_constituents`` per instance. + # + # Each instance owns its own slot ``(inst)%items(:)``: the + # property objects are independent across instances so that + # ``ccpp_register_constituents`` can ``set_const_index`` on each + # without conflicting with other instances. The outer wrapper + # array is allocated once on first call (any instance); each + # instance then appends each scheme's constituents into its slot, + # calling every scheme's ``_register`` EXACTLY ONCE (register may + # allocate persistent module state, so the earlier two-pass + # count+copy that called it twice broke non-idempotent schemes such + # as ``prescribed_aerosols_register``). The state-machine guard + # above this block ensures each instance runs the fill at most once. + # + # auto-clone-constituents: the legacy shim contributes one + # additional ``%instantiate`` per consumer-side + # ``is_constituent`` arg with no register-phase source. Those + # synthesised entries are appended to the same per-instance buffer + # slot after the scheme-registered entries. + const_scheme_names = {scheme_name for scheme_name, _ in suite_res.constituent_register_calls} + buf = '{}_dynamic_constituents'.format(suite_name) + n_auto_clone = len(suite_res.auto_cloned_constituents) + + # Allocate the outer wrapper array on first call (any instance). + lines.append( + '{}if (.not. allocated({})) then'.format(i2, buf) + ) + lines.append('{}allocate({}({}))'.format( + i2 + _INDENT, buf, ninstances_arg, + )) + lines.append('{}end if'.format(i2)) + lines.append('') + + # Single pass: call each constituent scheme's _register EXACTLY ONCE + # and append its returned array to this instance's slot. Start from + # an empty slot and grow it; intrinsic assignment of the constituent + # array constructor deep-copies each entry (same assignment the old + # copy loop used element-wise). + lines.append( + "{}! Pack each scheme's constituents (register run once each).".format(i2) + ) + lines.append('{}allocate({}({})%items(0))'.format(i2, buf, inst_idx)) + for _gname, resolved_call in _register_calls(suite_res): + if resolved_call.scheme_name in const_scheme_names: + _emit_register_call(resolved_call, i2, errflg_local, lines) + lines.append( + '{0}{1}({2})%items = [{1}({2})%items, scheme_consts]'.format( + i2, buf, inst_idx, + ) + ) + lines.append('{}deallocate(scheme_consts)'.format(i2)) + # auto-clone-constituents: reserve n_auto_clone trailing slots after + # the scheme-registered entries, then instantiate into them. + if n_auto_clone > 0: + lines.append('') + lines.append( + '{}! auto-clone-constituents: reserve + instantiate synthesised entries'.format( + i2, + ) + ) + lines.append( + '{}num_consts = size({}({})%items, 1)'.format(i2, buf, inst_idx) + ) + if has_consts: + # Grow the existing scheme-registered slot by n_auto_clone. + lines.append( + '{}call move_alloc({}({})%items, scheme_consts)'.format( + i2, buf, inst_idx, + ) + ) + lines.append( + '{}allocate({}({})%items(num_consts + {}))'.format( + i2, buf, inst_idx, n_auto_clone, + ) + ) + lines.append( + '{0}if (num_consts > 0) {1}({2})%items(1:num_consts) = ' + 'scheme_consts(1:num_consts)'.format(i2, buf, inst_idx) + ) + lines.append( + '{}if (allocated(scheme_consts)) deallocate(scheme_consts)'.format(i2) + ) + else: + # No scheme-registered entries: just size the slot directly. + lines.append('{}deallocate({}({})%items)'.format(i2, buf, inst_idx)) + lines.append( + '{}allocate({}({})%items({}))'.format( + i2, buf, inst_idx, n_auto_clone, + ) + ) + for entry in suite_res.auto_cloned_constituents: + _emit_auto_clone_instantiate( + entry, buf, inst_idx, i2, errflg_local, errmsg_local, lines, + ) + lines.append('') + # Emit any non-constituent register calls in addition (always, per instance). + for _gname, resolved_call in _register_calls(suite_res): + if resolved_call.scheme_name not in const_scheme_names: + _emit_register_call(resolved_call, i2, errflg_local, lines) + else: + # No constituent merge — emit register calls in suite-XML order. + for _gname, resolved_call in _register_calls(suite_res): + _emit_register_call(resolved_call, i2, errflg_local, lines) + + lines.append('') + lines.append( + '{}ccpp_suite_state({}) = CCPP_SUITE_REGISTERED'.format(i2, inst_idx) + ) + lines.append('') + lines.append('{}end subroutine {}'.format(i1, sub_name)) + return lines + + +def _init_lines( + suite_name: str, + suite_res: SuiteResolution, + host_dict=None, +) -> List[str]: + """Generate the ``_init`` framework-setup subroutine lines. + + Per-instance lifecycle: every call passes ``instance_number`` (when the host + declares it). Requires the suite to be in ``CCPP_SUITE_REGISTERED`` (i.e. + ``ccpp_register`` was called). The body: + + 1. Verifies state is ``REGISTERED``; idempotent skip if already + ``FRAMEWORK_INITIALIZED``; error otherwise. + 2. Calls each group ``state_alloc`` routine (idempotent first-call alloc). + 3. Calls the suite-data ``init_fields`` routine which allocates inner + allocatable suite-data fields using suite-owned dim values that may have + been written during the register phase. + 4. Sets ``ccpp_suite_state(instance_number) = CCPP_SUITE_FRAMEWORK_INITIALIZED``. + + Minimal signature: ``(instance_number, number_of_instances, errmsg, + errflg)`` -- the instance pair is included only when the host declares it. + """ + sub_name = '{}_init'.format(suite_name) + i1 = _INDENT + i2 = _INDENT * 2 + + # ``number_of_instances`` is now a paired control variable; it arrives + # as a dummy argument rather than via ``use ``. + ninstances_entry = host_dict.get('number_of_instances') if host_dict else None + ninstances_local = ninstances_entry.local_name if ninstances_entry else None + ninstances_arg = ninstances_local if ninstances_local else '1' + + inst_local = _instance_local(host_dict) + inst_idx = _instance_idx(host_dict) + + errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') or 'errflg' + errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') or 'errmsg' + + sig_args: List[str] = [] + if inst_local: + sig_args.append(inst_local) + if ninstances_local: + sig_args.append(ninstances_local) + sig_args += [errmsg_local, errflg_local] + + lines: List[str] = [''] + lines.append('{}subroutine {}({})'.format(i1, sub_name, ', '.join(sig_args))) + + # USE: suite_data init_fields routine when this suite owns any vars; + # constituent object (from host module) for pointer binding; + # suite-level scheme module + per-arg host modules. + # (``number_of_instances`` used to be USE'd here; now it's a dummy arg.) + extra_uses: Dict[str, Set[str]] = {} + if suite_res.suite_vars: + data_mod = 'ccpp_{}_data'.format(suite_name) + init_fields = 'suite_data_init_fields' + extra_uses.setdefault(data_mod, set()).add(init_fields) + if suite_res.suite_init_call is not None: + _add_call_uses(extra_uses, suite_res.suite_init_call) + for mod in sorted(extra_uses): + syms = ', '.join(sorted(extra_uses[mod])) + lines.append('{}use {}, only: {}'.format(i2, mod, syms)) + + lines.append('') + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + if ninstances_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, ninstances_local)) + lines += [ + '{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local), + '{}integer, intent(out) :: {}'.format(i2, errflg_local), + ] + extra_in = [ninstances_local] if ninstances_local else None + trace_lines = emit_trace_block( + sub_name, [], i2, + instance_local=inst_local, extra_in_names=extra_in, + ) + if trace_lines: + lines.append('') + lines.extend(trace_lines) + lines += [ + '', + "{}{} = ''".format(i2, errmsg_local), + '{}{} = 0'.format(i2, errflg_local), + '', + ] + + # State guard: must be in REGISTERED state (or already INITIALIZED — idempotent). + sub_label = '{}_init'.format(suite_name) + lines += [ + '{}if (.not. allocated(ccpp_suite_state)) then'.format(i2), + "{} {} = '{}: ccpp_register has not been called'".format( + i2, errmsg_local, sub_label + ), + '{} {} = 1'.format(i2, errflg_local), + '{} return'.format(i2), + '{}end if'.format(i2), + '{}if (ccpp_suite_state({}) == CCPP_SUITE_FRAMEWORK_INITIALIZED) return'.format( + i2, inst_idx + ), + '{}if (ccpp_suite_state({}) /= CCPP_SUITE_REGISTERED) then'.format( + i2, inst_idx + ), + "{} {} = '{}: invalid suite state (expected REGISTERED)'".format( + i2, errmsg_local, sub_label + ), + '{} {} = 1'.format(i2, errflg_local), + '{} return'.format(i2), + '{}end if'.format(i2), + '', + ] + + # Group state allocators (idempotent). + for resolved_group in suite_res.groups: + alloc_sub = '{}_state_alloc'.format(resolved_group.group_name) + lines.append('{}call {}({}, {}, {})'.format( + i2, alloc_sub, ninstances_arg, errmsg_local, errflg_local + )) + lines.append('{}if ({} /= 0) return'.format(i2, errflg_local)) + + # Allocate inner suite-data allocatable fields for this instance. + if suite_res.suite_vars: + init_fields = 'suite_data_init_fields' + lines.append('{}call {}({}, {}, {})'.format( + i2, init_fields, inst_idx, errmsg_local, errflg_local + )) + lines.append('{}if ({} /= 0) return'.format(i2, errflg_local)) + + # Constituent state binding is owned by the host_constituents module + # under option A — the host calls ccpp_initialize_constituents separately + # to bind ccpp_constituents/ccpp_constituent_tendencies and populate the + # index_of_ integers. + + # Suite-level scheme call (if declared in the SDF). Runs once + # per ``_init`` invocation, after all group state allocators + # have populated their state arrays, and before the suite-state + # transition to FRAMEWORK_INITIALIZED. Errflg check follows the call. + if suite_res.suite_init_call is not None: + lines.append('') + from generator.group_cap import _emit_one_call + _emit_one_call(suite_res.suite_init_call, i2, lines) + + lines += [ + '', + '{}ccpp_suite_state({}) = CCPP_SUITE_FRAMEWORK_INITIALIZED'.format( + i2, inst_idx + ), + '', + '{}end subroutine {}'.format(i1, sub_name), + ] + return lines + + +def _final_lines( + suite_name: str, + suite_res: SuiteResolution, + host_dict=None, +) -> List[str]: + """Generate the ``_final`` framework-teardown subroutine lines. + + Per-instance lifecycle: + + 1. Errors if ``ccpp_suite_state`` is not allocated. + 2. Per-instance idempotent skip: returns immediately if this instance's slot + is already ``CCPP_SUITE_UNREGISTERED``. + 3. Calls suite-data ``final_fields`` for this instance to deallocate inner + allocatable suite-data fields (when this suite owns any). + 4. Sets ``ccpp_suite_state(instance_number) = CCPP_SUITE_UNREGISTERED``. + 5. Last-to-leave dealloc: when every slot is ``UNREGISTERED`` after the + flip, calls each group ``state_dealloc`` and the suite ``state_dealloc`` + (which also tears down the suite_data DDT array). + + Signature: ``(instance_number, number_of_instances, errmsg, errflg)`` + when the host declares the multi-instance pair, else + ``(errmsg, errflg)``. ``number_of_instances`` is carried for API + symmetry with ``_register`` / ``_init``; the framework + does not consume it at final time. + """ + sub_name = '{}_final'.format(suite_name) + i1 = _INDENT + i2 = _INDENT * 2 + + inst_local = _instance_local(host_dict) + inst_idx = _instance_idx(host_dict) + ninstances_entry = host_dict.get('number_of_instances') if host_dict else None + ninstances_local = ninstances_entry.local_name if ninstances_entry else None + + errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') or 'errflg' + errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') or 'errmsg' + + sig_args: List[str] = [] + if inst_local: + sig_args.append(inst_local) + if ninstances_local: + sig_args.append(ninstances_local) + sig_args += [errmsg_local, errflg_local] + + lines: List[str] = [''] + lines.append('{}subroutine {}({})'.format(i1, sub_name, ', '.join(sig_args))) + + final_uses: Dict[str, Set[str]] = {} + if suite_res.suite_vars: + data_mod = 'ccpp_{}_data'.format(suite_name) + final_fields = 'suite_data_final_fields' + final_uses.setdefault(data_mod, set()).add(final_fields) + + # If we registered constituents, the per-suite buffer (owned by + # ccpp_host_constituents) is torn down here in the last-to-leave block. + if suite_res.constituent_register_calls: + buf = '{}_dynamic_constituents'.format(suite_name) + final_uses.setdefault('ccpp_host_constituents', set()).add(buf) + + # Suite-level scheme call (if declared in the SDF) — pull + # in the scheme module and any host modules its args reference. + if suite_res.suite_final_call is not None: + _add_call_uses(final_uses, suite_res.suite_final_call) + + for mod in sorted(final_uses): + syms = ', '.join(sorted(final_uses[mod])) + lines.append('{}use {}, only: {}'.format(i2, mod, syms)) + + lines.append('') + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + if ninstances_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, ninstances_local)) + lines += [ + '{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local), + '{}integer, intent(out) :: {}'.format(i2, errflg_local), + ] + extra_in = [ninstances_local] if ninstances_local else None + trace_lines = emit_trace_block( + sub_name, [], i2, + instance_local=inst_local, extra_in_names=extra_in, + ) + if trace_lines: + lines.append('') + lines.extend(trace_lines) + lines += [ + '', + "{}{} = ''".format(i2, errmsg_local), + '{}{} = 0'.format(i2, errflg_local), + '', + # ``_final`` is silently idempotent: a repeat call must return + # cleanly with ``errflg=0``. After the first call's last-to-leave + # teardown the state array is deallocated, so the unallocated path is + # the normal post-final state — silent-return rather than error. The + # per-instance ``UNREGISTERED`` skip covers any other instance that + # was already finalized before the last-to-leave dealloc fired. + '{}if (.not. allocated(ccpp_suite_state)) return'.format(i2), + '{}if (ccpp_suite_state({}) == CCPP_SUITE_UNREGISTERED) return'.format( + i2, inst_idx + ), + '', + ] + + # Deallocate inner suite-data fields if this instance was past REGISTERED. + if suite_res.suite_vars: + final_fields = 'suite_data_final_fields' + lines.append( + '{}if (ccpp_suite_state({}) == CCPP_SUITE_FRAMEWORK_INITIALIZED) then'.format( + i2, inst_idx + ) + ) + lines.append('{} call {}({}, {}, {})'.format( + i2, final_fields, inst_idx, errmsg_local, errflg_local + )) + lines.append('{} if ({} /= 0) return'.format(i2, errflg_local)) + lines.append('{}end if'.format(i2)) + lines.append('') + + # Suite-level scheme call (if declared in the SDF). Runs + # once per ``_final`` invocation, before the suite-state + # transition to UNREGISTERED. Errflg check follows the call. + if suite_res.suite_final_call is not None: + from generator.group_cap import _emit_one_call + _emit_one_call(suite_res.suite_final_call, i2, lines) + + lines.append( + '{}ccpp_suite_state({}) = CCPP_SUITE_UNREGISTERED'.format(i2, inst_idx) + ) + lines.append('') + lines.append( + '{}if (all(ccpp_suite_state == CCPP_SUITE_UNREGISTERED)) then'.format(i2) + ) + for resolved_group in suite_res.groups: + dealloc_sub = '{}_state_dealloc'.format(resolved_group.group_name) + lines.append('{} call {}({}, {})'.format( + i2, dealloc_sub, errmsg_local, errflg_local + )) + lines.append('{} if ({} /= 0) return'.format(i2, errflg_local)) + suite_dealloc_sub = '{}_suite_state_dealloc'.format(suite_name) + lines.append('{} call {}({}, {})'.format( + i2, suite_dealloc_sub, errmsg_local, errflg_local + )) + lines.append('{} if ({} /= 0) return'.format(i2, errflg_local)) + # Constituent OBJ teardown lives in ccpp_deallocate_dynamic_constituents + # (the host calls it per instance + last-to-leave dealloc). The + # per-suite ``_dynamic_constituents`` buffer, however, is + # tied to THIS suite's lifecycle — populated by ``_register`` + # under the suite-cap state guard — so it must be deallocated here + # in the last-to-leave block, not in the constituent-deallocate + # routine. Otherwise the next ``ccpp_register`` short-circuits on + # the state guard without re-filling the buffer. + if suite_res.constituent_register_calls: + buf = '{}_dynamic_constituents'.format(suite_name) + lines.append('{} if (allocated({})) deallocate({})'.format( + i2, buf, buf, + )) + lines += [ + '{}end if'.format(i2), + '', + '{}end subroutine {}'.format(i1, sub_name), + ] + return lines + + +def _physics_dispatch_lines( + suite_name: str, + phase: str, + suite_res: SuiteResolution, + host_dict=None, +) -> List[str]: + """Generate a ``_physics_`` dispatch subroutine. + + The subroutine signature is derived entirely from the host's ``type=control`` + metadata (all control variables except ``suite_name``, which is consumed at the + static API dispatch level). When ``group_name`` is in the control table the + body uses a ``select case`` dispatch; otherwise all groups are called + unconditionally. + """ + sub_name = '{}_physics_{}'.format(suite_name, phase) + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + + # Suite-level signature: all ctrl vars excluding suite_name. + ctrl_entries = _ctrl_entries_for_signature(host_dict, exclude={'suite_name'}) + ctrl_local_names = [e.local_name for e in ctrl_entries] + + # Determine if group_name is in the control table. + group_name_entry = next( + (e for e in ctrl_entries if e.standard_name == 'group_name'), None + ) + has_group_name = group_name_entry is not None + + # Group-level args passed when calling group cap subroutines. + group_ctrl_entries = _ctrl_entries_for_signature( + host_dict, exclude={'suite_name', 'group_name'} + ) + group_ctrl_local = [e.local_name for e in group_ctrl_entries] + + lines: List[str] = [] + lines.append('') + + # Subroutine signature. + if ctrl_local_names: + lines.append('{}subroutine {}( &'.format(i1, sub_name)) + for i, lname in enumerate(ctrl_local_names): + sep = ', &' if i < len(ctrl_local_names) - 1 else ')' + lines.append('{} {}{}'.format(i1, lname, sep)) + else: + lines.append('{}subroutine {}()'.format(i1, sub_name)) + + # Dummy argument declarations. + lines.append('') + for entry in ctrl_entries: + # Character control dummies always use len=* so the host's specific + # length doesn't propagate into the generated signature. + kind = 'len=*' if entry.type.strip().lower() == 'character' else entry.kind + t = _fortran_type_str(entry.type, kind) + dim = _dim_decl(entry.dimensions) + intent = _intent_clause(_ctrl_intent_for(entry.standard_name)) + lines.append( + '{}{}{}{} :: {}'.format(i2, t, intent, dim, entry.local_name) + ) + + # Trace block: references every intent(in)/inout control dummy so that + # strict compilers don't flag any of them as unused. + trace_lines = emit_trace_block(sub_name, ctrl_entries, i2) + if trace_lines: + lines.append('') + lines.extend(trace_lines) + + lines.append('') + + # Initialize error reporting vars before any work, then guard on the + # per-instance suite state. + errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') + errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') + if errflg_local and errmsg_local: + lines.append("{}{} = ''".format(i2, errmsg_local)) + lines.append('{}{} = 0'.format(i2, errflg_local)) + lines.append('') + + inst_idx = _instance_idx(host_dict) + sub_label = '{}_physics_{}'.format(suite_name, phase) + if phase == 'final': + # ``physics_final`` is silently idempotent: a repeat call (or a + # call issued after ``ccpp_final``) must return cleanly with + # ``errflg=0`` rather than erroring. The group-level guard + # handles the per-group skip when ``ccpp_final`` has not been + # called; the two checks below cover the post-``ccpp_final`` + # cases (state array deallocated on the last instance, or set + # to ``UNREGISTERED`` on any other instance). + lines += [ + '{}if (.not. allocated(ccpp_suite_state)) return'.format(i2), + '{}if (ccpp_suite_state({}) == CCPP_SUITE_UNREGISTERED) return'.format( + i2, inst_idx + ), + ] + else: + lines += [ + '{}if (.not. allocated(ccpp_suite_state)) then'.format(i2), + "{} {} = '{}: ccpp_register has not been called'".format( + i2, errmsg_local, sub_label + ), + '{} {} = 1'.format(i2, errflg_local), + '{} return'.format(i2), + '{}end if'.format(i2), + ] + lines += [ + '{}if (ccpp_suite_state({}) /= CCPP_SUITE_FRAMEWORK_INITIALIZED) then'.format( + i2, inst_idx + ), + "{} {} = '{}: invalid suite state'".format( + i2, errmsg_local, sub_label + ), + '{} {} = 1'.format(i2, errflg_local), + '{} return'.format(i2), + '{}end if'.format(i2), + '', + ] + + def _emit_group_call(resolved_group, indent): + # Group phase subroutines are always emitted (so the per-group state + # machine transitions through every phase), so we always dispatch. + cap_sub = '{}_{}'.format(resolved_group.group_name, phase) + if group_ctrl_local: + lines.append('{}call {}( &'.format(indent, cap_sub)) + for idx, lname in enumerate(group_ctrl_local): + sep = ', &' if idx < len(group_ctrl_local) - 1 else ')' + lines.append('{} {}{}'.format(indent, lname, sep)) + else: + lines.append('{}call {}()'.format(indent, cap_sub)) + # Stop and propagate on the first group error. Each group phase + # subroutine resets ``errflg = 0`` on entry, so without this guard a + # ``group_name='all'`` dispatch would let a LATER group's success + # overwrite an EARLIER group's failure -- the error (and its message) + # would be silently masked and only resurface downstream as an + # "invalid group state" when ``run`` finds the failed group never + # reached ``IN_TIMESTEP``. Mirrors the per-scheme call guards. + if errflg_local: + lines.append('{}if ({} /= 0) return'.format(indent, errflg_local)) + + if has_group_name: + grp_local = group_name_entry.local_name + lines.append('{}select case(trim({}))'.format(i2, grp_local)) + # '' or 'all' → call all groups. + lines.append("{}case('', 'all')".format(i2)) + for resolved_group in suite_res.groups: + _emit_group_call(resolved_group, i3) + # Individual group cases. + for resolved_group in suite_res.groups: + lines.append("{}case('{}')".format(i2, resolved_group.group_name)) + _emit_group_call(resolved_group, i3) + # case default: anything other than '', 'all', or a known group + # is a runtime error — caller asked for a group this suite + # doesn't define. Without ccpp_error_code/_message in the host + # control table there's nowhere to write the message, so skip + # emission rather than silently swallow. + if errflg_local and errmsg_local: + sub_label = '{}_physics_{}'.format(suite_name, phase) + lines.append('{}case default'.format(i2)) + lines.append('{}{} = 1'.format(i3, errflg_local)) + lines.append( + "{}{} = '{}: unknown group: ' // trim({})".format( + i3, errmsg_local, sub_label, grp_local, + ) + ) + lines.append('{}return'.format(i3)) + lines.append('{}end select'.format(i2)) + else: + # No group_name control var: call all groups unconditionally. + for resolved_group in suite_res.groups: + _emit_group_call(resolved_group, i2) + + lines.append('') + lines.append('{}end subroutine {}'.format(i1, sub_name)) + return lines + + +def _suite_state_alloc_lines( + suite_name: str, + has_suite_vars: bool, +) -> List[str]: + """Generate the ``_suite_state_alloc`` subroutine. + + Idempotent allocator for the per-instance suite state array and the + suite-owned DDT array. Inner allocatable fields inside the DDT are NOT + allocated here — that happens in ``ccpp__suite_data_init_fields``, + called from ``_init`` after register-phase scheme calls have set + any suite-owned scalar dimensions. + """ + sub_name = '{}_suite_state_alloc'.format(suite_name) + data_alloc = 'suite_data_alloc' + data_mod = 'ccpp_{}_data'.format(suite_name) + i1 = _INDENT + i2 = _INDENT * 2 + lines = [ + '', + '{}subroutine {}(number_of_instances, errmsg, errflg)'.format(i1, sub_name), + ] + if has_suite_vars: + lines.append('{}use {}, only: {}'.format(i2, data_mod, data_alloc)) + lines += [ + '', + '{}integer, intent(in) :: number_of_instances'.format(i2), + '{}character(len=*), intent(out) :: errmsg'.format(i2), + '{}integer, intent(out) :: errflg'.format(i2), + '', + "{}errmsg = ''".format(i2), + '{}errflg = 0'.format(i2), + '{}if (allocated(ccpp_suite_state)) return'.format(i2), + '{}allocate(ccpp_suite_state(number_of_instances))'.format(i2), + '{}ccpp_suite_state(:) = CCPP_SUITE_UNREGISTERED'.format(i2), + ] + if has_suite_vars: + lines += [ + '{}call {}(number_of_instances, errmsg, errflg)'.format(i2, data_alloc), + '{}if (errflg /= 0) return'.format(i2), + ] + lines += [ + '', + '{}end subroutine {}'.format(i1, sub_name), + ] + return lines + + +def _suite_state_dealloc_lines( + suite_name: str, + has_suite_vars: bool, +) -> List[str]: + """Generate the ``_suite_state_dealloc`` subroutine.""" + sub_name = '{}_suite_state_dealloc'.format(suite_name) + data_dealloc = 'suite_data_dealloc' + data_mod = 'ccpp_{}_data'.format(suite_name) + i1 = _INDENT + i2 = _INDENT * 2 + lines = [ + '', + '{}subroutine {}(errmsg, errflg)'.format(i1, sub_name), + ] + if has_suite_vars: + lines.append('{}use {}, only: {}'.format(i2, data_mod, data_dealloc)) + lines += [ + '', + '{}character(len=*), intent(out) :: errmsg'.format(i2), + '{}integer, intent(out) :: errflg'.format(i2), + '', + "{}errmsg = ''".format(i2), + '{}errflg = 0'.format(i2), + ] + if has_suite_vars: + lines += [ + '{}call {}(errmsg, errflg)'.format(i2, data_dealloc), + '{}if (errflg /= 0) return'.format(i2), + ] + lines += [ + '{}if (allocated(ccpp_suite_state)) deallocate(ccpp_suite_state)'.format(i2), + '', + '{}end subroutine {}'.format(i1, sub_name), + ] + return lines + + +######################################################################## +# Module generator +######################################################################## + +def _generate_suite_cap( + suite_name: str, + suite_res: SuiteResolution, + scheme_store: SchemeStore, + host_dict=None, + trace: bool = False, +) -> List[str]: + """Generate the full ``ccpp__cap.F90`` module source lines. + + Parameters + ---------- + suite_name : str + suite_res : SuiteResolution + scheme_store : SchemeStore + host_dict : dict, optional + Flat host+control variable dictionary. When provided, ``number_of_instances`` + and ``instance_number`` are used for multi-instance state array sizing and + indexing. + + Returns + ------- + list of str (no trailing newlines) + """ + mod_name = 'ccpp_{}_cap'.format(suite_name) + lines: List[str] = [] + + # Module header. + lines.append( + '! ccpp_{}_cap.F90 -- generated by ccpp_capgen, do not edit'.format( + suite_name + ) + ) + lines.append('module {}'.format(mod_name)) + lines.append('') + + # USE statements: one per group cap (all phase + state subroutines). + # Group cap subroutine names are short (``_`` etc.) so + # the mangled global ``_mp_`` stays under Intel's ~90-char + # limit even for long suite/group name combinations. + use_lines: List[str] = [] + for resolved_group in suite_res.groups: + group_cap_mod = 'ccpp_{}_{}_{}'.format(suite_name, resolved_group.group_name, 'cap') + syms_list = [ + '{}_{}'.format(resolved_group.group_name, p) + for p in _PHYSICS_PHASES + ] + syms_list.append('{}_state_alloc'.format(resolved_group.group_name)) + syms_list.append('{}_state_dealloc'.format(resolved_group.group_name)) + use_lines.append('{}use {}, only: {}'.format( + _INDENT, group_cap_mod, ', '.join(syms_list) + )) + + # Trace block writes to error_unit; ensure the USE is present. + ensure_error_unit_use(use_lines, _INDENT) + lines.extend(use_lines) + + lines.append('') + lines.append('{}implicit none'.format(_INDENT)) + lines.append('{}private'.format(_INDENT)) + lines.append('') + + # Public declarations: all framework lifecycle and physics phase entry + # points are always emitted. ``ccpp_register`` is mandatory in the new + # design — even an empty register phase fires the state transition. + pub_subs = [] + pub_subs.append('{}_register'.format(suite_name)) + pub_subs.append('{}_init'.format(suite_name)) + for phase in _PHYSICS_PHASES: + pub_subs.append('{}_physics_{}'.format(suite_name, phase)) + pub_subs.append('{}_final'.format(suite_name)) + pub_subs.append('{}_suite_state_alloc'.format(suite_name)) + pub_subs.append('{}_suite_state_dealloc'.format(suite_name)) + + for sub in pub_subs: + lines.append('{}public :: {}'.format(_INDENT, sub)) + + lines.append('') + lines.append('{}integer, private, parameter :: CCPP_SUITE_UNREGISTERED = 0'.format(_INDENT)) + lines.append('{}integer, private, parameter :: CCPP_SUITE_REGISTERED = 1'.format(_INDENT)) + lines.append('{}integer, private, parameter :: CCPP_SUITE_FRAMEWORK_INITIALIZED = 2'.format(_INDENT)) + lines.append('{}integer, private, allocatable :: ccpp_suite_state(:)'.format(_INDENT)) + lines.extend(emit_module_gate(trace, _INDENT)) + lines.append('') + lines.append('contains') + + # Subroutines. Order: register, init, physics_*, final, state_alloc/dealloc. + lines.extend(_register_lines(suite_name, suite_res, host_dict)) + lines.extend(_init_lines(suite_name, suite_res, host_dict)) + for phase in _PHYSICS_PHASES: + lines.extend(_physics_dispatch_lines(suite_name, phase, suite_res, host_dict)) + lines.extend(_final_lines(suite_name, suite_res, host_dict)) + + has_suite_vars = bool(suite_res.suite_vars) + lines.extend(_suite_state_alloc_lines(suite_name, has_suite_vars)) + lines.extend(_suite_state_dealloc_lines(suite_name, has_suite_vars)) + + lines.append('') + lines.append('end module {}'.format(mod_name)) + return lines + + +######################################################################## +# Public API +######################################################################## + +def write_suite_cap( + suite_name: str, + suite_res: SuiteResolution, + scheme_store: SchemeStore, + output_root: str, + host_dict=None, + logger: Optional[logging.Logger] = None, + trace: bool = False, +) -> str: + """Write ``ccpp__cap.F90`` to *output_root*. + + Parameters + ---------- + suite_name : str + suite_res : SuiteResolution + scheme_store : SchemeStore + output_root : str + Output directory (created if absent). + host_dict : dict, optional + Flat host+control dictionary for multi-instance support. + + Returns + ------- + str + Absolute path of the written file. + """ + os.makedirs(output_root, exist_ok=True) + filename = 'ccpp_{}_cap.F90'.format(suite_name) + out_path = os.path.join(output_root, filename) + + lines = _generate_suite_cap( + suite_name, suite_res, scheme_store, host_dict, trace=trace, + ) + with open_if_changed(out_path, logger=logger) as fh: + fh.write('\n'.join(lines) + '\n') + return out_path diff --git a/capgen/generator/suite_data.py b/capgen/generator/suite_data.py new file mode 100644 index 00000000..1864c37e --- /dev/null +++ b/capgen/generator/suite_data.py @@ -0,0 +1,494 @@ +#!/usr/bin/env python3 + +"""Generate the suite data module ``ccpp__data.F90``.""" + +import logging +import os +from typing import Dict, List, Optional + +from metadata.parse_tools import CCPPError, open_if_changed +from generator.suite_resolver import SuiteVar + +_INDENT = ' ' + +_INTRINSICS = frozenset({ + 'real', 'integer', 'character', 'logical', 'complex', 'double precision' +}) + +# Framework-provided constituent count dimension. A suite-owned variable may be +# dimensioned by number_of_ccpp_constituents; unlike host/suite dims its extent +# is owned by the framework's per-instance constituent object +# (``ccpp_model_constituents_obj(i)%num_layer_vars``) declared in module +# ``ccpp_host_constituents``. Mirrors the constants of the same name in +# generator.suite_resolver. +_CONST_NUM_STD = 'number_of_ccpp_constituents' +_CONST_OBJ_VAR = 'ccpp_model_constituents_obj' +_CONST_OBJ_MODULE = 'ccpp_host_constituents' +_CONST_NUM_MEMBER = 'num_layer_vars' + + +def _type_str(type_: str, kind: str) -> str: + """Return the Fortran type clause for a SuiteVar field. + + >>> _type_str('real', 'kind_phys') + 'real(kind=kind_phys)' + >>> _type_str('real', '') + 'real' + >>> _type_str('my_ddt', '') + 'type(my_ddt)' + >>> _type_str('character', 'len=512') + 'character(len=512)' + """ + t = type_.strip() + if t.lower() not in _INTRINSICS and not t.lower().startswith('external:'): + if not t.lower().startswith('type('): + t = 'type({})'.format(t) + if kind: + if t.lower().startswith('character'): + return 'character({})'.format(kind) + return '{}(kind={})'.format(t, kind) + return t + + +def _collect_dim_uses( + suite_vars: Dict[str, SuiteVar], + host_dict, +) -> Dict[str, List[str]]: + """Return {module_name: [local_name, ...]} for dimension variables of suite vars. + + Host-module variables (``is_control=False``) are included keyed by their + declaring module. Control variables and suite-owned dimensions are + excluded — suite-owned dim scalars (set during ``_register``) are accessed + via ``ccpp_suite_data(i)%`` directly within the same module so no + USE statement is needed for them. Control variables cannot be dimensions + of suite-owned data because they are not available at ``init_fields`` + time when allocations happen. + + Raises CCPPError if a dimension is found in host_dict but is a control + variable. Suite-owned dimensions (not in host_dict) are silently skipped. + """ + uses: Dict[str, List] = {} + seen: set = set() + suite_var_std_names = set(suite_vars.keys()) + for suite_var in sorted(suite_vars.values(), key=lambda v: v.standard_name): + for dim_std in suite_var.dimensions: + if dim_std in seen: + continue + seen.add(dim_std) + # Suite-owned dim → no USE needed (same module access). + if dim_std in suite_var_std_names: + continue + # Framework constituent count → extent comes from the per-instance + # constituent object, not the host or a suite scalar. USE its + # module so init_fields can reference the count member. + if dim_std == _CONST_NUM_STD: + uses.setdefault(_CONST_OBJ_MODULE, []) + if _CONST_OBJ_VAR not in uses[_CONST_OBJ_MODULE]: + uses[_CONST_OBJ_MODULE].append(_CONST_OBJ_VAR) + continue + if host_dict is None: + raise CCPPError( + "Suite-owned variable '{}' has dimension '{}' but no host " + "metadata was provided to resolve it".format( + suite_var.standard_name, dim_std + ) + ) + entry = host_dict.get(dim_std) + if entry is None: + raise CCPPError( + "Suite-owned variable '{}' dimension '{}' not found in " + "host metadata or in suite-owned variables".format( + suite_var.standard_name, dim_std + ) + ) + if entry.is_control: + raise CCPPError( + "Suite-owned variable '{}' dimension '{}' is a control " + "variable; suite data must use host-module or " + "suite-owned dimensions".format(suite_var.standard_name, dim_std) + ) + mod = entry.module_name + if mod not in uses: + uses[mod] = [] + if entry.local_name not in uses[mod]: + uses[mod].append(entry.local_name) + return uses + + +def _dim_local_expr(dim_std: str, suite_vars: Dict[str, SuiteVar], host_dict) -> str: + """Return the Fortran expression to use as one allocation dimension token. + + Suite-owned scalars: ``ccpp_suite_data(i)%`` (in the alloc loop + context where ``i`` is the instance index variable). Host-owned: just the + local name. The caller substitutes ``i`` for the instance index variable. + """ + if dim_std in suite_vars: + return 'ccpp_suite_data(i)%{}'.format(suite_vars[dim_std].local_name) + # Framework constituent count: extent from the per-instance constituent + # object (``i`` is the instance index in the init_fields alloc context). + if dim_std == _CONST_NUM_STD: + return '{}(i)%{}'.format(_CONST_OBJ_VAR, _CONST_NUM_MEMBER) + entry = host_dict.get(dim_std) if host_dict else None + if entry is None: + raise CCPPError( + "Cannot resolve suite-owned dimension '{}' from host or " + "suite vars".format(dim_std) + ) + return entry.local_name + + +def _collect_ddt_uses( + suite_vars: Dict[str, SuiteVar], + ddt_module_map: Optional[Dict[str, str]], +) -> Dict[str, List[str]]: + """Return ``{module_name: [ddt_type, ...]}`` for DDT types referenced + by suite-owned variables. + + A type is treated as a DDT when it is neither a Fortran intrinsic nor + an ``external:module:typename`` reference. The DDT type → module + mapping must be supplied by the caller (typically built via + :func:`metadata.variable_resolver.build_ddt_module_map`). + + Raises CCPPError if a DDT-typed suite variable references a type that + is missing from *ddt_module_map*. + """ + uses: Dict[str, List[str]] = {} + seen: set = set() + for suite_var in sorted(suite_vars.values(), key=lambda v: v.standard_name): + t = suite_var.type_.strip() + tlow = t.lower() + if tlow in _INTRINSICS or tlow.startswith('external:'): + continue + if tlow.startswith('type('): + t = t[t.index('(') + 1:t.rindex(')')].strip() + if t in seen: + continue + seen.add(t) + if ddt_module_map is None or t not in ddt_module_map: + raise CCPPError( + "Suite-owned variable '{}' has DDT type '{}' but its " + "defining Fortran module is unknown. Declare it explicitly " + "with 'module_name = ' in the '{}' DDT's " + "[ccpp-table-properties], or co-locate the DDT table with a " + "scheme/host/control table in the same .meta file.".format( + suite_var.standard_name, t, t, + ) + ) + mod = ddt_module_map[t] + uses.setdefault(mod, []) + if t not in uses[mod]: + uses[mod].append(t) + return uses + + +def _generate_suite_data( + suite_name: str, + suite_vars: Dict[str, SuiteVar], + host_dict=None, + ddt_module_map: Optional[Dict[str, str]] = None, +) -> List[str]: + """Generate the ``ccpp__data.F90`` module source lines. + + >>> lines = _generate_suite_data('mysuite', {}) + >>> 'module ccpp_mysuite_data' in lines + True + >>> any('ccpp_mysuite_data_t' in l for l in lines) + True + """ + mod_name = 'ccpp_{}_data'.format(suite_name) + type_name = 'ccpp_{}_data_t'.format(suite_name) + # Short Fortran symbols; the module ``ccpp__data`` already + # namespaces these routines at link time, keeping the mangled global + # symbol ``_mp_`` well under Intel's ~90-char limit. + alloc_sub = 'suite_data_alloc' + dealloc_sub = 'suite_data_dealloc' + init_fields_sub = 'suite_data_init_fields' + final_fields_sub = 'suite_data_final_fields' + i1 = _INDENT + i2 = _INDENT * 2 + lines: List[str] = [] + + lines.append( + '! ccpp_{}_data.F90 -- generated by ccpp_capgen, do not edit'.format( + suite_name + ) + ) + lines.append('module {}'.format(mod_name)) + lines.append('') + + # USE ccpp_kinds for any kind parameters referenced in suite-var + # declarations (e.g. ``real(kind=kind_phys)``). + kind_names = sorted({ + suite_var.kind for suite_var in suite_vars.values() + if suite_var.kind and not suite_var.kind.startswith('len=') + }) + if kind_names: + lines.append( + '{}use ccpp_kinds, only: {}'.format(i1, ', '.join(kind_names)) + ) + + # USE the defining module of each DDT type referenced by suite-owned + # variables so that ``type() :: `` declarations are valid. + ddt_uses = _collect_ddt_uses(suite_vars, ddt_module_map) + for mod in sorted(ddt_uses): + types = sorted(ddt_uses[mod]) + lines.append( + '{}use {}, only: {}'.format(i1, mod, ', '.join(types)) + ) + if kind_names or ddt_uses: + lines.append('') + + lines.append('{}implicit none'.format(i1)) + lines.append('{}private'.format(i1)) + lines.append('') + + # DDT type — allocatable fields use deferred-shape colons. + # + # ``TARGET`` is not a valid component attribute in Fortran; instead + # the module-level ``ccpp_suite_data(:)`` array below carries + # ``TARGET`` so that every ``ccpp_suite_data(i)%component(...)`` + # subobject is a valid pointer-assignment target. This is what + # the group cap needs to do ``ptr%ptr => ccpp_suite_data(i)%fld(...)`` + # for optional-arg passing and transformation temporaries. + lines.append('{}type, public :: {}'.format(i1, type_name)) + if suite_vars: + for suite_var in sorted(suite_vars.values(), key=lambda v: v.standard_name): + t = _type_str(suite_var.type_, suite_var.kind) + if suite_var.dimensions: + rank = len(suite_var.dimensions) + deferred = '({})'.format(','.join([':'] * rank)) + lines.append( + '{}{}, allocatable :: {}{}'.format( + i2, t, suite_var.local_name, deferred, + ) + ) + else: + lines.append('{}{} :: {}'.format(i2, t, suite_var.local_name)) + else: + lines.append('{}! (no suite-owned variables)'.format(i2)) + lines.append('{}end type {}'.format(i1, type_name)) + lines.append('') + + # Module-level allocatable instance array (one per model instance). + # ``TARGET`` makes every subobject (component access, array section, + # nested DDT field, ...) a valid pointer-assignment target. Without + # it Fortran rejects ``ptr => ccpp_suite_data(i)%fld(...)`` with + # "Pointer assignment target is neither TARGET nor POINTER". + lines.append( + '{}type({}), allocatable, target, public :: ccpp_suite_data(:)'.format( + i1, type_name, + ) + ) + + # Alloc/dealloc subroutines are only generated when host_dict is provided + # (the integration path). Unit tests that call without host_dict get the + # type definition and allocatable array but no subroutines. + if suite_vars and host_dict is not None: + lines.append('') + lines.append('{}public :: {}'.format(i1, alloc_sub)) + lines.append('{}public :: {}'.format(i1, dealloc_sub)) + lines.append('{}public :: {}'.format(i1, init_fields_sub)) + lines.append('{}public :: {}'.format(i1, final_fields_sub)) + lines.append('') + lines.append('contains') + + sorted_svs = sorted(suite_vars.values(), key=lambda v: v.standard_name) + + # ---- suite_data_alloc: only allocate the DDT array --------------- + # Inner allocatable fields are NOT allocated here because their + # dimensions may depend on suite-owned scalars set during the + # register phase. Inner allocations live in init_fields, called + # from _init after all _register calls have run. + lines.append('') + lines.append( + '{}subroutine {}(number_of_instances, errmsg, errflg)'.format(i1, alloc_sub) + ) + lines += [ + '', + '{}integer, intent(in) :: number_of_instances'.format(i2), + '{}character(len=*), intent(out) :: errmsg'.format(i2), + '{}integer, intent(out) :: errflg'.format(i2), + '', + "{}errmsg = ''".format(i2), + '{}errflg = 0'.format(i2), + '{}if (allocated(ccpp_suite_data)) return'.format(i2), + '{}allocate(ccpp_suite_data(number_of_instances))'.format(i2), + '', + '{}end subroutine {}'.format(i1, alloc_sub), + ] + + # ---- suite_data_dealloc: only deallocate the DDT array ----------- + lines.append('') + lines.append( + '{}subroutine {}(errmsg, errflg)'.format(i1, dealloc_sub) + ) + lines += [ + '', + '{}character(len=*), intent(out) :: errmsg'.format(i2), + '{}integer, intent(out) :: errflg'.format(i2), + '', + "{}errmsg = ''".format(i2), + '{}errflg = 0'.format(i2), + '{}if (.not. allocated(ccpp_suite_data)) return'.format(i2), + '{}deallocate(ccpp_suite_data)'.format(i2), + '', + '{}end subroutine {}'.format(i1, dealloc_sub), + ] + + # ---- suite_data_init_fields: allocate inner fields per instance -- + dim_uses = _collect_dim_uses(suite_vars, host_dict) + lines.append('') + lines.append( + '{}subroutine {}(i, errmsg, errflg)'.format(i1, init_fields_sub) + ) + for mod in sorted(dim_uses): + syms = ', '.join(sorted(dim_uses[mod])) + lines.append('{}use {}, only: {}'.format(i2, mod, syms)) + lines += [ + '', + '{}integer, intent(in) :: i'.format(i2), + '{}character(len=*), intent(out) :: errmsg'.format(i2), + '{}integer, intent(out) :: errflg'.format(i2), + '', + "{}errmsg = ''".format(i2), + '{}errflg = 0'.format(i2), + ] + for suite_var in sorted_svs: + # Allocatable suite vars are owned-and-allocated by the producing + # scheme itself (its dummy is declared ``allocatable, intent(out)``, + # so the callee performs the allocation). The suite must NOT + # pre-allocate them here -- doing so is at best redundant (the + # scheme's intent(out) auto-deallocates on entry) and at worst + # wrong (its extent may not be known until the scheme runs). The + # suite still OWNS the storage, so final_fields below deallocates + # it regardless of who allocated it. + if suite_var.dimensions and not suite_var.allocatable: + dim_exprs = [ + _dim_local_expr(d, suite_vars, host_dict) + for d in suite_var.dimensions + ] + lines.append( + '{}allocate(ccpp_suite_data(i)%{}({}))'.format( + i2, suite_var.local_name, ', '.join(dim_exprs) + ) + ) + lines += [ + '', + '{}end subroutine {}'.format(i1, init_fields_sub), + ] + + # ---- suite_data_final_fields: deallocate inner fields per inst --- + lines.append('') + lines.append( + '{}subroutine {}(i, errmsg, errflg)'.format(i1, final_fields_sub) + ) + lines += [ + '', + '{}integer, intent(in) :: i'.format(i2), + '{}character(len=*), intent(out) :: errmsg'.format(i2), + '{}integer, intent(out) :: errflg'.format(i2), + '', + "{}errmsg = ''".format(i2), + '{}errflg = 0'.format(i2), + ] + # Deallocate ALL dimensioned fields here -- including allocatable + # ones that a scheme allocated itself. The storage is a component of + # the suite-owned ``ccpp_suite_data`` DDT, so the suite owns its + # teardown; the ``if (allocated(...))`` guard makes this safe whether + # the scheme allocated it, never ran, or already freed it. + for suite_var in sorted_svs: + if suite_var.dimensions: + lines.append( + '{}if (allocated(ccpp_suite_data(i)%{})) ' + 'deallocate(ccpp_suite_data(i)%{})'.format( + i2, suite_var.local_name, suite_var.local_name + ) + ) + lines += [ + '', + '{}end subroutine {}'.format(i1, final_fields_sub), + ] + + lines.append('') + lines.append('end module {}'.format(mod_name)) + return lines + + +def write_suite_data( + suite_name: str, + suite_vars: Dict[str, SuiteVar], + output_root: str, + host_dict=None, + ddt_module_map: Optional[Dict[str, str]] = None, + logger: Optional[logging.Logger] = None, +) -> str: + """Write ``ccpp__data.F90`` to *output_root*.""" + os.makedirs(output_root, exist_ok=True) + filename = 'ccpp_{}_data.F90'.format(suite_name) + out_path = os.path.join(output_root, filename) + lines = _generate_suite_data(suite_name, suite_vars, host_dict, + ddt_module_map=ddt_module_map) + with open_if_changed(out_path, logger=logger) as fh: + fh.write('\n'.join(lines) + '\n') + return out_path + + +def _generate_suite_meta( + suite_name: str, + suite_vars: Dict[str, SuiteVar], +) -> List[str]: + """Generate metadata lines for ``ccpp__data.meta``. + + The file documents all suite-owned variables in the standard ``.meta`` + format so that downstream tools can inspect what each suite provides. + The ``_data`` suffix matches the companion Fortran file + ``ccpp__data.F90``, satisfying the ``.meta`` ↔ ``.F90`` pairing + convention. + + >>> lines = _generate_suite_meta('mysuite', {}) + >>> lines[0].startswith('!') + True + >>> any('ccpp_mysuite_data' in l for l in lines) + True + """ + mod_name = 'ccpp_{}_data'.format(suite_name) + i1 = _INDENT + lines: List[str] = [] + lines.append( + '! ccpp_{}_data.meta -- generated by ccpp_capgen, do not edit'.format(suite_name) + ) + lines.append('[ccpp-table-properties]') + lines.append('{}name = {}'.format(i1, mod_name)) + lines.append('{}type = suite'.format(i1)) + lines.append('') + lines.append('[ccpp-arg-table]') + lines.append('{}name = {}'.format(i1, mod_name)) + lines.append('{}type = suite'.format(i1)) + for suite_var in sorted(suite_vars.values(), key=lambda v: v.standard_name): + lines.append('') + lines.append('[ {} ]'.format(suite_var.local_name)) + lines.append('{}standard_name = {}'.format(i1, suite_var.standard_name)) + lines.append('{}long_name = {}'.format(i1, suite_var.standard_name)) + lines.append('{}units = {}'.format(i1, suite_var.units)) + dim_str = '({})'.format(', '.join(suite_var.dimensions)) if suite_var.dimensions else '()' + lines.append('{}dimensions = {}'.format(i1, dim_str)) + lines.append('{}type = {}'.format(i1, suite_var.type_)) + if suite_var.kind: + lines.append('{}kind = {}'.format(i1, suite_var.kind)) + return lines + + +def write_suite_meta( + suite_name: str, + suite_vars: Dict[str, SuiteVar], + output_root: str, + logger: Optional[logging.Logger] = None, +) -> str: + """Write ``ccpp__data.meta`` to *output_root* and return its path.""" + os.makedirs(output_root, exist_ok=True) + filename = 'ccpp_{}_data.meta'.format(suite_name) + out_path = os.path.join(output_root, filename) + lines = _generate_suite_meta(suite_name, suite_vars) + with open_if_changed(out_path, logger=logger) as fh: + fh.write('\n'.join(lines) + '\n') + return out_path diff --git a/capgen/generator/suite_resolver.py b/capgen/generator/suite_resolver.py new file mode 100644 index 00000000..006dc7d2 --- /dev/null +++ b/capgen/generator/suite_resolver.py @@ -0,0 +1,3014 @@ +#!/usr/bin/env python3 + +"""Variable matching and call-site resolution for the cap code generator. + +Resolves every scheme argument against the flat host/control dictionary built by +:func:`metadata.variable_resolver.build_flat_host_dict`, discovers suite-owned +(interstitial) variables, detects unit/kind transformations, and builds the +complete call-site information needed by :mod:`generator.group_cap`. + +Variable matching rules (Section 8.4 of the redesign spec) +----------------------------------------------------------- +For each standard name requested by a scheme argument: + +1. **Found in host/control dict** → direct reference; check units/kind for + transformation. +2. **Not found, first use is ``intent(out)``** → suite-owned variable; add to + suite data, generate declaration in ``ccpp__data.F90``. +3. **Not found, first use is ``intent(in)`` or ``intent(inout)``** → code + generation error: variable used before it is provided. +4. **Already in suite data (from a prior scheme)** → reference suite data path; + check units/kind for transformation. + +Dimension indexing rules (Section 9.2) +--------------------------------------- +Each entry in the ``dimensions`` list of a host/suite variable is either a +bare standard name (``'vertical_layer_dimension'``) or an explicit +``lower:upper`` range (``'ccpp_constant_one:horizontal_dimension'``, +``'bot_idx:vertical_interface_dimension'``). Bare names are normalised to +``ccpp_constant_one:`` before processing. + +After normalisation the upper-bound standard name drives dispatch: + +- Registered scalar-index dim (see + ``metadata.registered_dimensions.SCALAR_INDEX_DIMS``; currently + ``number_of_instances`` → ``instance_number``, + ``number_of_threads`` → ``thread_number``) → scalar extraction using + the paired index variable's local name. The scalar subscript is + already in the access path for DDT-component fields, but needed here + for a DDT instance variable itself when passed directly, and for any + flat-array dim that hits the same registered name. +- ``horizontal_dimension`` → + ``:`` (all phases). The lower bound must resolve to + ``1`` (i.e. be ``ccpp_constant_one`` or the integer literal ``1``). +- Everything else → ``:`` where both bounds are + resolved from *host_dict* or as integer literals. + +Transform cases (Section 10.3) +------------------------------- +Four cases, determined by ``optional`` and whether units/kind differ: + +====== ================ ============ +Case optional? transform? +====== ================ ============ +1 no no +2 yes no +3 no yes +4 yes yes +====== ================ ============ +""" + +import hashlib +import re +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Set, Tuple, Union + +from metadata.parse_tools import CCPPError, FORTRAN_CONDITIONAL_REGEX +from metadata.registered_dimensions import ( + SCALAR_INDEX_DIMS, + scalar_index_for, + is_scalar_index_dim, +) +from metadata.variable_resolver import HostVarEntry, _resolve_subscript +# dim-aliases: transient GFS-physics shim (delete this import and the +# canonical() call in _canonical_dim when the shim is removed). +from metadata import dim_aliases +# auto-clone-constituents: transient legacy shim (delete this import +# and the AutoCloneEntry / _collect_auto_clone_entries touchpoints +# when the shim is removed). +from metadata import auto_clone_constituents + +# Dimension standard names that map to horizontal loop bounds. The +# legacy spelling ``horizontal_loop_extent`` is rejected at parse time +# (see ``_FORBIDDEN_DIMENSION_NAMES`` in ccpp_capgen.py) or rewritten +# by the ``--legacy-mode`` shim, so it can never appear here. +_HORIZ_LOOP_DIMS: frozenset = frozenset({ + 'horizontal_dimension', +}) + +# Standard names for horizontal loop bounds and full horizontal dimension. +_HORIZ_BEGIN_STD = 'horizontal_loop_begin' +_HORIZ_END_STD = 'horizontal_loop_end' +_HORIZ_DIM_STD = 'horizontal_dimension' +_INSTANCE_NUM_STD = 'instance_number' + +# Vertical-dimension standard names (used by the vertical-flip transform +# when host and scheme metadata disagree on the ``top_at_one`` attribute). +_VDIM_STDS: frozenset = frozenset({ + 'vertical_layer_dimension', + 'vertical_interface_dimension', +}) + +# Physics scheme phases that operate on the per-call horizontal slice and +# therefore receive (ub - lb + 1) when a scheme asks for a scalar +# horizontal_dimension. Register is excluded: it runs at suite-cap level +# with the minimal framework signature (no loop bounds available). +_PHYSICS_PHASES: frozenset = frozenset({ + 'init', 'timestep_init', 'run', 'timestep_final', 'final', +}) + +# Framework constant whose Fortran value is always 1. Used as the implicit +# lower bound when a dimension string carries no explicit range, and is the +# only non-integer lower bound accepted for horizontal dimensions. +_CCPP_CONSTANT_ONE = 'ccpp_constant_one' + +# Type marker for register-phase constituent registration args. A scheme +# arg with this type, ``intent=out``, in the ``register`` phase is recognised +# as the per-scheme constituent array for the two-pass merge into the host's +# ``ccpp_model_constituents_object``. +_CONST_PROP_TYPE = 'ccpp_constituent_properties_t' + +# Standard name the host model uses to expose its ``ccpp_model_constituents_t`` +# object via the ``type=host`` table (opt-in, only required when at least +# one register-phase scheme produces constituents). +_CONST_OBJ_STDNAME = 'ccpp_model_constituents_object' + +# Framework-provided constituent standard names. The suite cap owns +# these symbols (allocates/binds them at init time); schemes reference +# them like any other variable and the resolver routes them to a +# synthetic source category ``'constituent'``. +_CONST_BASE_ARRAY_STD = 'ccpp_constituents' +_CONST_TEND_ARRAY_STD = 'ccpp_constituent_tendencies' +_CONST_PROPS_ARRAY_STD = 'ccpp_constituent_properties' +_CONST_NUM_STD = 'number_of_ccpp_constituents' +_CONST_MINVAL_STD = 'ccpp_constituent_minimum_values' +_TEND_PREFIX = 'tendency_of_' +_INDEX_PREFIX = 'index_of_' + +# Fortran 2008 caps user-defined identifiers at 63 characters. Several +# CCPP standard-name conventions (notably CAM-SIMA's +# ``_wrt_moist_air_and_condensed_water`` constituent suffix) produce +# base names ≥55 chars, which blow the limit once the ``index_of_`` +# prefix is prepended. The helper below mangles overlong names to a +# deterministic 63-char form so every emitter and resolver call site +# sees the same symbol; short names are returned unchanged. +_FORTRAN_ID_LIMIT = 63 + + +def _index_symbol_name(base_std_name: str) -> str: + """Return the Fortran local name for ``index_of_``. + + Identity for inputs whose ``index_of_`` form fits Fortran's 63-char + identifier limit; otherwise truncates the base and appends a short + SHA-1 hash so distinct std-names map to distinct symbols. The + chosen layout is:: + + index_of__<8-hex-sha1> + + where ``max_base_len = 63 - len('index_of_') - 1 - 8 = 45``. All + emit/reference sites (host_constituents.py public/declaration/ + reset/const_index/init-guard, suite_resolver.py auto-provisioned + subscript, Path 1a call_expr) MUST route through this helper to + keep the symbol consistent within a single capgen run. The + underlying std_name is still passed to ``const_index`` as a + string literal, so the framework lookup keys remain unchanged -- + only the Fortran-side mapping symbol is mangled. + + Examples + -------- + >>> _index_symbol_name('water_vapor') + 'index_of_water_vapor' + >>> name = _index_symbol_name( + ... 'cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water') + >>> len(name) <= 63 + True + >>> name.startswith('index_of_') + True + >>> name == _index_symbol_name( + ... 'cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water') + True + """ + full = _INDEX_PREFIX + base_std_name + if len(full) <= _FORTRAN_ID_LIMIT: + return full + import hashlib + sha8 = hashlib.sha1(base_std_name.encode('utf-8')).hexdigest()[:8] + # 63 - len('index_of_') - 1 (sep) - 8 (sha) = 45 + max_base_len = _FORTRAN_ID_LIMIT - len(_INDEX_PREFIX) - 1 - 8 + return '{}{}_{}'.format( + _INDEX_PREFIX, base_std_name[:max_base_len], sha8, + ) + +# Std names directly satisfied by host-constituents-module-owned symbols. +_FRAMEWORK_CONST_STDS = frozenset({ + _CONST_BASE_ARRAY_STD, + _CONST_TEND_ARRAY_STD, + _CONST_PROPS_ARRAY_STD, + _CONST_NUM_STD, + _CONST_MINVAL_STD, +}) + +# Per-instance constituent object name in ccpp_host_constituents. Schemes +# access constituent state through ``(inst_num)%``. +_CONST_OBJ_VAR = 'ccpp_model_constituents_obj' + +# Mapping from framework-named std_name → DDT member. Used to translate +# scheme args declaring one of these framework names into the matching +# per-instance access expression. +_FRAMEWORK_NAME_TO_MEMBER = { + _CONST_BASE_ARRAY_STD: 'vars_layer', + _CONST_TEND_ARRAY_STD: 'vars_layer_tend', + _CONST_PROPS_ARRAY_STD: 'const_metadata', + _CONST_NUM_STD: 'num_layer_vars', + _CONST_MINVAL_STD: 'vars_minvalue', +} + + +# Single host-wide module that owns the constituent object, the +# framework-shared pointers, the per-suite dynamic-constituent buffers, +# and the host-facing constituent API. All suite caps USE this module +# for their constituent symbol references. +_HOST_CONST_MOD = 'ccpp_host_constituents' + + +def _constituent_module_name(suite_name: str) -> str: + """Return the module name that owns the host-wide constituent state. + + Constant across suites: in capgen (option A, matching original + capgen) the constituent object is host-wide, not suite-local. + """ + return _HOST_CONST_MOD + + +######################################################################## +# Unit conversion look-up +######################################################################## + +def _normalize_unit_string(unit: str) -> str: + """Canonicalise a unit string so that bare positive exponents carry an + explicit ``+`` sign. + + The CF / udunits conventions allow either ``m2`` or ``m+2`` to denote + "metres squared". The two forms are equivalent, but downstream code + treats unit strings as opaque tokens and compares them with ``==``. + Without normalisation, a host declaring ``m2 s-2`` and a scheme + declaring ``m+2 s-2`` would be flagged as a unit mismatch. + + Normalisation rule: a letter immediately followed by an unsigned + positive integer is rewritten as ``letter+integer``. Existing + ``letter+N`` and ``letter-N`` forms are left unchanged. + """ + return re.sub(r'([A-Za-z])(\d+)', r'\1+\2', unit) + + +def _unit_to_id(unit: str) -> str: + """Convert a unit string to the Python identifier fragment used in + :mod:`metadata.unit_conversion`. + + The input is first normalised by :func:`_normalize_unit_string` so + that bare and explicit positive exponents collapse to the same form. + + Rules (after normalisation): + + * Spaces → ``_`` + * ``letter-N`` → ``letter_minus_N`` + * ``letter+N`` → ``letter_plus_N`` + """ + result = _normalize_unit_string(unit).replace(' ', '_') + result = re.sub(r'([A-Za-z])([+])(\d+)', r'\1_plus_\3', result) + result = re.sub(r'([A-Za-z])(-)(\d+)', r'\1_minus_\3', result) + return result + + +def find_unit_conversion(from_unit: str, to_unit: str): + """Return the conversion formula callable, or ``None`` if unavailable. + + The formula callable takes no arguments and returns a format string + where ``{var}`` is the Fortran expression to convert and ``{kind}`` + is the kind suffix (``_kind_phys`` or ``''``). + + Input unit strings are normalised by :func:`_normalize_unit_string` + before the equality check and the function-name lookup, so equivalent + forms (``m2`` and ``m+2``) compare equal and resolve to the same + conversion entry. + """ + from_norm = _normalize_unit_string(from_unit) + to_norm = _normalize_unit_string(to_unit) + if from_norm == to_norm: + return None + from metadata import unit_conversion as _uc + fn_name = '{}__to__{}'.format(_unit_to_id(from_norm), _unit_to_id(to_norm)) + return getattr(_uc, fn_name, None) + + +def _apply_transform_formula(formula_fn, var_expr: str, kind: str) -> str: + """Apply a unit-conversion formula callable. + + Parameters + ---------- + formula_fn : callable + Returned by :func:`find_unit_conversion`. + var_expr : str + Fortran expression for the source variable. + kind : str + Kind parameter name (e.g. ``'kind_phys'``), or ``''``. + + Returns + ------- + str + Fortran expression for the converted value. + """ + kind_suffix = '_{}'.format(kind) if kind else '' + return formula_fn().format(kind=kind_suffix, var=var_expr) + + +#: Map from CCPP metadata ``type =`` to the Fortran kind-cast intrinsic. +#: Only the numeric types that legitimately carry distinct kinds in +#: CCPP physics are mapped; everything else (logical, character, DDT) +#: is rejected by :func:`_kind_cast_expr` because either the kind itself +#: is dimensioned differently (character ``len=``) or kinds don't apply. +_KIND_CAST_INTRINSIC = { + 'real': 'real', + 'integer': 'int', + 'complex': 'cmplx', +} + + +def _kind_cast_expr( + var_type: str, + var_expr: str, + target_kind: str, + local: str, + std_name: str, + scheme_name: str, +) -> str: + """Build a Fortran expression that casts *var_expr* to *target_kind*. + + Used when host and scheme metadata differ in ``kind`` only -- no unit + conversion, no vertical flip -- so the transformation temporary + needs an explicit precision cast. Without this the temp would be + declared but never assigned (see :func:`_resolve_one_arg`). + + Returns a Fortran expression like ``real(con_pi, kind=kind_phys)``. + + Raises + ------ + CCPPError + When *var_type* is not one of the numeric types listed in + :data:`_KIND_CAST_INTRINSIC` (kinds on logical / character are + handled via separate metadata pathways, and DDT kinds don't + apply -- a kind mismatch on those types indicates the metadata + is wrong rather than something the cap should bridge). + """ + intrinsic = _KIND_CAST_INTRINSIC.get(var_type.strip().lower()) + if intrinsic is None: + raise CCPPError( + "Variable '{}' (standard_name='{}', scheme '{}'): host and " + "scheme metadata differ in 'kind' but the variable type " + "'{}' has no defined kind-cast intrinsic. Kind differences " + "are supported for real / integer / complex only; fix the " + "metadata so the kinds match.".format( + local, std_name, scheme_name, var_type, + ) + ) + return '{}({}, kind={})'.format(intrinsic, var_expr, target_kind) + + +######################################################################## +# Dimension subscript helpers +######################################################################## + +def _format_available_std_names( + host_dict: Dict[str, HostVarEntry], + suite_vars: Optional[Dict[str, 'SuiteVar']] = None, + near: Optional[str] = None, +) -> str: + """Build a sorted listing of every standard name the resolver can see. + + Used in error messages when a lookup fails. Each line is + `` []`` where source is ``control``, + ``host: ``, or ``suite: ``. The final list + is sorted alphabetically (case-insensitive); when *near* is a + misspelled or mis-cased candidate, close matches surface first + under a separate "did you mean" header so the user spots the + typo quickly. + """ + rows: List[Tuple[str, str]] = [] + for std, entry in host_dict.items(): + if entry.is_control: + rows.append((std, 'control')) + elif entry.module_name: + rows.append((std, 'host: {}'.format(entry.module_name))) + else: + rows.append((std, 'host')) + if suite_vars: + for std, suite_var in suite_vars.items(): + rows.append((std, 'suite: {}'.format(suite_var.suite_module_name))) + rows.sort(key=lambda t: t[0]) + + if not rows: + return '\n (host_dict and suite_vars are both empty)' + + width = max(len(s) for s, _ in rows) + fmt = ' {{:<{}}} [{{}}]'.format(width) + + sections: List[str] = [] + if near: + import difflib + candidates = difflib.get_close_matches( + near, [s for s, _ in rows], n=5, cutoff=0.6, + ) + if not candidates: + # Try a case-insensitive direct hit (the most common cause: + # mixed-case in metadata vs lower-cased standard name). + low = near.lower() + candidates = [s for s, _ in rows if s == low] + if candidates: + sections.append('Did you mean (close matches to {!r}):'.format(near)) + sections.extend( + fmt.format(s, src) + for s, src in rows if s in candidates + ) + sections.append('') + + sections.append('Available standard names ({} entries):'.format(len(rows))) + sections.extend(fmt.format(s, src) for s, src in rows) + return '\n' + '\n'.join(sections) + + +def _resolve_single_bound( + bound: str, + host_dict: Dict[str, HostVarEntry], + used: Set[str], + suite_vars: Optional[Dict[str, 'SuiteVar']] = None, +) -> Optional[str]: + """Resolve one dimension bound token to a Fortran expression. + + Recognises, in order: + + 1. ``ccpp_constant_one`` — the framework constant equal to ``1``. + 2. Any integer literal — returned as a string unchanged. + 3. A standard name present in *host_dict* — returns the local Fortran name + and records the standard name in *used*. + 4. A standard name present in *suite_vars* — returns the suite data access + path (e.g. ``ccpp_suite_data(inst)%dim_inter``) and records the standard + name in *used*. Suite-owned scalars set during ``_register`` are read + here as dimension bounds in later phases. + + Returns ``None`` when the bound cannot be resolved. + """ + if bound == _CCPP_CONSTANT_ONE: + return '1' + try: + return str(int(bound)) + except ValueError: + pass + entry = host_dict.get(bound) + if entry is not None: + used.add(bound) + # Use ``access_path``, not ``local_name``: for plain module- + # level host vars these are identical, but for DDT-component + # vars (e.g. ``physics%Model%levs`` with std_name + # ``vertical_layer_dimension``) the full DDT walk is required + # so the emitted subscript references the actual storage and + # the USE statement (which walks back to the root via + # ``_root_symbol``) imports the right top-level symbol. + # ``_render_value_expr`` (a) resolves baked + # ``(instance_number)`` / ``(thread_number)`` placeholders and + # (b) re-attaches any literal subscript stripped from a + # ``local_name = foo(1)``-style declaration. + return _render_value_expr(entry, host_dict) + if suite_vars: + suite_var = suite_vars.get(bound) + if suite_var is not None: + used.add(bound) + return suite_var.access_path + return None + + +def _build_call_subscript( + dimensions: List[str], + phase: str, + host_dict: Dict[str, HostVarEntry], + suite_vars: Optional[Dict[str, 'SuiteVar']] = None, + flip_vertical: bool = False, +) -> Tuple[str, Set[str]]: + """Build the Fortran subscript string for a variable's dimension list. + + Returns the subscript string (empty for scalars or ``'(s1, s2, ...)'`` for + arrays) and the set of dimension standard names that were resolved via + *host_dict* (needed for USE-statement generation). + + Parameters + ---------- + dimensions : list of str + Ordered dimension standard names from the host/suite variable entry. + phase : str + Current scheme phase (affects horizontal subscripting). + host_dict : dict + Flat host+control variable dictionary. + flip_vertical : bool + When ``True``, every vertical-dimension entry is emitted with + reverse stride (``::-1`` instead of ``:``). + Used by the vertical-flip transform when host and scheme disagree + on ``top_at_one``. + + Returns + ------- + tuple (subscript_str, used_std_names) + + Raises + ------ + CCPPError + If a dimension standard name cannot be resolved. + """ + if not dimensions: + return '', set() + + parts: List[str] = [] + used: Set[str] = set() + for dim in dimensions: + part, u = _one_dim_part(dim, phase, host_dict, suite_vars=suite_vars, + flip_vertical=flip_vertical) + parts.append(part) + used.update(u) + return '({})'.format(', '.join(parts)), used + + +def _one_dim_part( + dim: str, + phase: str, + host_dict: Dict[str, HostVarEntry], + suite_vars: Optional[Dict[str, 'SuiteVar']] = None, + flip_vertical: bool = False, +) -> Tuple[str, Set[str]]: + """Return the Fortran subscript expression for one dimension entry. + + A dimension entry is either a bare standard name (``'vertical_layer_dimension'``) + or an explicit lower:upper range (``'ccpp_constant_one:horizontal_dimension'``, + ``'bot_idx:vertical_interface_dimension'``). Bare names are normalised + internally to ``ccpp_constant_one:`` before processing. + + Rules applied after normalisation: + + * Upper bound is a registered scalar-index dim (see + :data:`metadata.registered_dimensions.SCALAR_INDEX_DIMS`) → scalar + subscript using the paired index variable's local name (e.g. + ``instance_number``, ``thread_number``). + * Upper bound in :data:`_HORIZ_LOOP_DIMS` → ``lb:ub`` (loop bounds). + Lower bound **must** resolve to ``'1'`` (i.e. be ``ccpp_constant_one`` + or the integer literal ``1``); any other value is an error. + * Everything else → resolve both bounds from *host_dict* or as integer + literals and return ``lower_expr:upper_expr``. + + When *flip_vertical* is True and the dimension is a vertical-axis + dimension (its upper bound is in :data:`_VDIM_STDS`), the bounds are + emitted in reverse-stride form ``::-1`` so the + array section reads (and writes) the vertical axis bottom-to-top + instead of top-to-bottom. This is how the host-side access expression + is rendered when host metadata declares ``top_at_one = .true.`` but + the scheme expects bottom-at-one (or vice versa). + + Returns ``(expr, used_std_names)`` where *expr* is the subscript token + and *used_std_names* is the set of standard names consumed from *host_dict*. + """ + used: Set[str] = set() + + # Normalise to range: bare name → ccpp_constant_one:name + if ':' not in dim: + lower_str = _CCPP_CONSTANT_ONE + upper_str = dim + else: + lower_str, upper_str = dim.split(':', 1) + lower_str = lower_str.strip() + upper_str = upper_str.strip() + + # Framework-provided constituent count dimension. Any variable -- host, + # suite-owned, or scheme -- may be dimensioned by + # ``number_of_ccpp_constituents``. The framework owns the extent via the + # per-instance constituent object, so a *call subscript* passes the whole + # constituent axis. This mirrors :func:`_const_dim_part` (which only fires + # for framework-constituent args) and generalises the recognition to every + # other variable dimensioned by the count. There is no host scalar to USE, + # so ``used`` is left untouched. (Allocating a suite-owned var sized by + # this dim is handled separately in generator.suite_data, which resolves the + # extent to ``ccpp_model_constituents_obj(i)%num_layer_vars``.) + if upper_str == _CONST_NUM_STD: + return ':', used + + # Registered scalar-index dimension: collapse to the paired index + # variable's local Fortran name regardless of lower bound. See + # capgen/metadata/registered_dimensions.py for the contract. + idx_std = scalar_index_for(upper_str) + if idx_std is not None: + idx_entry = host_dict.get(idx_std) + if idx_entry is None: + raise CCPPError( + "Metadata references registered scalar-index dimension " + "'{dim}', which is paired with index variable '{idx}', " + "but the host has not declared '{idx}' in any type=control " + "or type=host table. Either declare '{idx}' as a scalar " + "integer in the host control/host metadata, or remove " + "the '{dim}' dimension from the affected metadata. See " + "capgen/metadata/registered_dimensions.py for the full " + "table of registered scalar-index pairings.".format( + dim=upper_str, idx=idx_std, + ) + ) + used.add(idx_std) + return idx_entry.local_name, used + + # Horizontal dimension: validate lower, return loop bounds + if upper_str in _HORIZ_LOOP_DIMS: + lower_expr = _resolve_single_bound(lower_str, host_dict, set()) + if lower_expr != '1': + raise CCPPError( + "Lower bound '{}' for horizontal dimension '{}' must be " + "1 or ccpp_constant_one".format(lower_str, upper_str) + ) + lb = host_dict.get(_HORIZ_BEGIN_STD) + ub = host_dict.get(_HORIZ_END_STD) + if lb is None or ub is None: + raise CCPPError( + "Dimension '{}' requires '{}' and '{}' in the host " + "metadata but they were not found".format( + dim, _HORIZ_BEGIN_STD, _HORIZ_END_STD + ) + ) + used.update({_HORIZ_BEGIN_STD, _HORIZ_END_STD, upper_str}) + return '{}:{}'.format(lb.local_name, ub.local_name), used + + # General range: resolve both bounds independently + lower_expr = _resolve_single_bound(lower_str, host_dict, used, + suite_vars=suite_vars) + if lower_expr is None: + raise CCPPError( + "Dimension lower bound '{}' in '{}' is not in the " + "host metadata or suite-owned variables.{}".format( + lower_str, dim, + _format_available_std_names(host_dict, suite_vars, near=lower_str), + ) + ) + upper_expr = _resolve_single_bound(upper_str, host_dict, used, + suite_vars=suite_vars) + if upper_expr is None: + if upper_str.startswith('vertical_'): + raise CCPPError( + "Vertical dimension '{}' is not in the host metadata.{}".format( + upper_str, + _format_available_std_names(host_dict, suite_vars, near=upper_str), + ) + ) + raise CCPPError( + "Dimension '{}' is not in the host metadata or suite-owned " + "variables.{}".format( + dim, + _format_available_std_names(host_dict, suite_vars, near=upper_str), + ) + ) + if flip_vertical and upper_str in _VDIM_STDS: + # Reverse-stride form for the vertical axis (top_at_one mismatch). + return '{}:{}:-1'.format(upper_expr, lower_expr), used + return '{}:{}'.format(lower_expr, upper_expr), used + + +def _build_merged_subscript( + host_dims: List[str], + local_subscript: List[str], + phase: str, + host_dict: Dict[str, HostVarEntry], + suite_vars: Optional[Dict[str, 'SuiteVar']] = None, + flip_vertical: bool = False, +) -> Tuple[str, Set[str]]: + """Build a call subscript merging a local-name template with host dimensions. + + Walk *local_subscript* tokens left to right: + + * ``':'`` — dimension placeholder: consume the next entry from *host_dims* + and emit the appropriate range expression (``lb:ub``, ``1:nlev``, etc.) + via :func:`_one_dim_part`. + * an integer literal — emitted verbatim. + * any other token — explicit index using a CCPP standard name: resolved + case-insensitively (standard names are case-insensitive) to the + corresponding local Fortran name from *host_dict* (or *suite_vars*). + The resolved standard name is added to the returned ``used`` set so the + group cap emits a ``use , only: `` for it. An + unresolved non-literal token is a metadata error: subscript indices in + a sliced ``local_name`` must be standard names with a defining source. + + When *local_subscript* is empty this is equivalent to calling + :func:`_build_call_subscript` directly. + + Example + ------- + local_subscript = [':', ':', 'index_of_water_vapor'] + host_dims = ['horizontal_dimension', 'vertical_layer_dimension'] + phase = 'run' + → '(lb:ub, 1:nlev, wv_idx)' + """ + if not local_subscript: + return _build_call_subscript(host_dims, phase, host_dict, + suite_vars=suite_vars, + flip_vertical=flip_vertical) + + dim_iter = iter(host_dims) + parts: List[str] = [] + used: Set[str] = set() + + for token in local_subscript: + token = token.strip() + if token == ':': + dim = next(dim_iter) + part, u = _one_dim_part(dim, phase, host_dict, + suite_vars=suite_vars, + flip_vertical=flip_vertical) + parts.append(part) + used.update(u) + elif token.isdigit(): + parts.append(token) + else: + key = token.lower() + entry = host_dict.get(key) + if entry is not None: + # Use ``access_path`` so DDT-component subscript indices + # (e.g. ``q(:,:,index_of_)`` where index_of_X lives + # on a DDT) resolve to the full DDT walk, not the bare + # leaf name. ``_render_value_expr`` (a) resolves baked + # ``(instance_number)``/``(thread_number)`` placeholders + # and (b) re-attaches any literal subscript stripped + # from a ``local_name = foo(1)``-style declaration. + parts.append(_render_value_expr(entry, host_dict)) + used.add(key) + elif suite_vars and key in suite_vars: + parts.append( + _substitute_scalar_idx(suite_vars[key].access_path, host_dict) + ) + used.add(key) + else: + raise CCPPError( + "Subscript index '{}' in a sliced local_name is not a " + "known CCPP standard name in the host or suite metadata; " + "subscript indices must be standard names with a " + "defining source so the cap can resolve and import " + "the corresponding local variable".format(token) + ) + + return '({})'.format(', '.join(parts)), used + + +def _dim_has_vertical(dim: str) -> bool: + """Return True if a dimension entry's upper bound is a vertical-axis + standard name. + + Accepts both the bare form (``'vertical_layer_dimension'``) and the + explicit ``lower:upper`` form (``'ccpp_constant_one:vertical_layer_dimension'``, + ``'bot_idx:vertical_interface_dimension'``). + """ + upper = dim.split(':', 1)[-1].strip() if ':' in dim else dim.strip() + return upper in _VDIM_STDS + + +def _canonical_dim(dim: str) -> str: + """Return a dimension entry in canonical ``lower:upper`` form for + identity comparison. + + Three spellings of the implicit/default lower bound all collapse + to a single representative: + + * bare ``foo`` (no explicit lower bound) + * ``1:foo`` (the integer literal one) + * ``ccpp_constant_one:foo`` (the standard name) + + Any *other* lower bound is distinct: ``2:foo`` is not the same + axis as ``1:foo``, ``bar:foo`` is not the same as ``1:foo``, etc. + Different lower bound describes a different sub-range and must + not compare equal. + + No name aliasing on the upper bound happens here by default. + ``horizontal_loop_extent`` and ``horizontal_dimension`` are + different names — the :func:`metadata.legacy_compat` shim is the + canonical place that rewrites the legacy name to the new one at + parse time when ``--legacy-mode`` is enabled. Without that shim + the two should never appear on opposite sides of a host/scheme + pairing. + + The one *opt-in* exception is the GFS dim-aliases shim + (:mod:`metadata.dim_aliases`, ``--gfs-dim-aliases``): when enabled + it collapses a small audited list of physically-equivalent + standard names (e.g. ``adjusted_vertical_layer_dimension_for_radiation`` + -> ``vertical_layer_dimension``) on the *upper bound only*, so + host and scheme metadata that use different historical spellings + of the same axis compare equal here. Variables keep their + original standard names elsewhere. + """ + if ':' in dim: + lower, upper = dim.split(':', 1) + else: + lower, upper = _CCPP_CONSTANT_ONE, dim + lower = lower.strip().lower() + upper = upper.strip().lower() + # Collapse the integer literal '1' and the standard name + # 'ccpp_constant_one' to a single representative so all three + # spellings of the default lower bound compare equal. + if lower == '1': + lower = _CCPP_CONSTANT_ONE + # dim-aliases: transient GFS-physics shim. No-op unless the + # ``--gfs-dim-aliases`` CLI flag has been passed. Only the upper + # bound is rewritten; lower bounds (loop-begin control vars, etc.) + # never alias. + upper = dim_aliases.canonical(upper) + return '{}:{}'.format(lower, upper) + + +def _substitute_scalar_idx( + expr: str, host_dict: Dict[str, HostVarEntry], +) -> str: + """Resolve registered scalar-index placeholders in a DDT access expr. + + :func:`metadata.variable_resolver._instance_subscript` bakes one + placeholder per registered scalar-index dim into the access path of + every HostVarEntry derived from a DDT-instance container. The + placeholders are *standard names* (e.g. ``instance_number``, + ``thread_number``) drawn from + :data:`metadata.registered_dimensions.SCALAR_INDEX_DIMS`; this + function resolves each to the host's actual Fortran local name at + codegen time. + + Resolution rules per placeholder: + + * Found in *host_dict* → substitute the host's ``local_name``. + * Absent from *host_dict* → substitute the literal ``1`` (consistent + with the ``instance_number`` paired-opt-in single-instance fallback; + length-1 internal arrays still address correctly). + + Multi-pair access paths like ``foo(instance_number, thread_number)`` + are handled in a single pass — every registered placeholder in the + expression is rewritten. + """ + # Optimization: skip the work when no placeholder could possibly be + # present. ``(`` is the cheapest distinguishing token. + if '(' not in expr: + return expr + out = expr + for idx_std in SCALAR_INDEX_DIMS.values(): + # Multiple placeholders may appear: as a sole subscript + # ``(idx_std)`` or as one of several ``(a, idx_std)``. Replace + # the bare std name token-wise, but only when it's clearly an + # index placeholder (preceded by ``(`` or ``, `` and followed by + # ``)`` or ``,``). In practice _instance_subscript only emits + # these inside a fresh subscript, so word-boundary replace is + # safe; we use re.sub to enforce the word boundary. + pattern = r'\b' + re.escape(idx_std) + r'\b' + entry = host_dict.get(idx_std) + replacement = entry.local_name if entry is not None else '1' + out = re.sub(pattern, replacement, out) + return out + + +# Backwards-compatibility shim — older code (and one external test) may +# still import the old name. Forward to the generalized impl. +_substitute_instance_idx = _substitute_scalar_idx + + +def _render_value_expr( + entry: HostVarEntry, + host_dict: Dict[str, HostVarEntry], +) -> str: + """Render *entry*'s full Fortran value-read expression. + + Combines two steps that callers usually need together: + + 1. Resolve any baked registered scalar-index placeholders in the + access path (``(instance_number)``, ``(thread_number)``) to the + host's local Fortran names via :func:`_substitute_scalar_idx`. + 2. Re-attach any literal subscript that was stripped from the + declared ``local_name`` at parse time (e.g. host metadata + declaring ``local_name = nstf_name(1)`` parses into + ``base='nstf_name'`` + ``local_subscript=['1']``; reading the + value requires re-appending ``(1)``). Std-name tokens inside + the subscript are themselves resolved to host local names via + :func:`metadata.variable_resolver._resolve_subscript`. + + Use this helper anywhere a host entry is rendered as a Fortran + expression in generator output (active-expression translation, + dimension-bound resolution, subscript-index tokens, subcycle + loop-count expressions, etc.). The scheme-arg base_expr + + _build_merged_subscript path is the exception — that path consumes + *entry.local_subscript* directly and interleaves it with scheme + dimensions, so it must not be pre-joined here. + """ + expr = _substitute_scalar_idx(entry.access_path, host_dict) + if entry.local_subscript: + sub = _resolve_subscript(', '.join(entry.local_subscript), host_dict) + expr = '{}({})'.format(expr, sub) + return expr + + +def _translate_active_expr(active: str, host_dict: Dict[str, HostVarEntry]) -> str: + """Translate standard names in an ``active`` expression to local Fortran. + + Standard-name identifiers are replaced with the host entry's full + Fortran access path (with any ``(instance_number)`` DDT-instance + template resolved to the host's actual local name). For free host + variables this collapses to ``entry.local_name``; for DDT-component + entries the substitution yields the fully qualified access path + (e.g. ``instance_data(instance)%opt_array_flag``). + """ + if not active: + return '' + + def _replace(m: re.Match) -> str: + word = m.group(0) + entry = host_dict.get(word) + if entry is None: + return word + return _render_value_expr(entry, host_dict) + + return FORTRAN_CONDITIONAL_REGEX.sub(_replace, active) + + +def _root_symbol(access_path: str) -> str: + """Return the root Fortran symbol from an access path. + + This is the part before any ``%`` or ``(``, which is the name that + appears in the ``use module, only: `` statement. + """ + return re.split(r'[%(]', access_path)[0] + + +_FORTRAN_ID_LIMIT_SV = 63 + + +def _unique_suite_field(desired: str, std_name: str, + suite_vars: Dict[str, 'SuiteVar']) -> str: + """Return a suite-data field name unique across existing suite vars. + + Two *distinct* suite-owned variables (different standard names) can be + first produced by scheme args that happen to share a local name -- e.g. + ``kdist`` for both ``longwave_gas_optics_object_for_RRTMGP`` and + ``shortwave_gas_optics_object_for_RRTMGP``, or ``hrate`` for the lw/sw + heating-rate tendencies. The generated ``ccpp__data`` DDT + declares one component per suite var named by this field, and the group + cap accesses it via the same name (``SuiteVar.access_path``), so the + field MUST be unique -- otherwise Fortran rejects the duplicate + component. + + Keep the bare name for the first occurrence (readable common case); + disambiguate a later collision with a short, deterministic + std_name-derived suffix (stable regardless of resolution order, since it + keys on the standard name rather than a counter). + """ + used = {sv.local_name for sv in suite_vars.values()} + if desired not in used: + return desired + suffix = hashlib.sha1(std_name.encode('utf-8')).hexdigest()[:8] + base = desired[:_FORTRAN_ID_LIMIT_SV - 1 - len(suffix)] + return '{}_{}'.format(base, suffix) + + +######################################################################## +# Data classes +######################################################################## + +@dataclass +class SuiteVar: + """A suite-owned variable discovered during variable resolution. + + Suite-owned variables are not provided by the host model; they are + first written by a scheme with ``intent(out)`` and then read by + subsequent schemes. They are declared in the generated + ``ccpp__data.F90`` module. + + Attributes + ---------- + standard_name : str + local_name : str + Scheme's local variable name that first produces this variable. + type_ : str + Fortran type string from the scheme metadata. + kind : str + Optional kind parameter. + units : str + dimensions : list of str + source_scheme : str + Name of the scheme that first declared it (intent out). + source_phase : str + """ + standard_name: str + local_name: str + type_: str + kind: str + units: str + dimensions: List[str] + source_scheme: str + source_phase: str + suite_module_name: str = '' + inst_access: str = '(1)' + allocatable: bool = False + + @property + def access_path(self) -> str: + """Fortran access expression in the suite data module.""" + return 'ccpp_suite_data{}%{}'.format(self.inst_access, self.local_name) + + @property + def module_name(self) -> str: + return self.suite_module_name if self.suite_module_name else 'ccpp_suite_data' + + +@dataclass +class ResolvedArg: + """One resolved argument at a scheme call site. + + Attributes + ---------- + standard_name : str + scheme_local_name : str + Keyword name for the Fortran call (from the scheme metadata ``[ name ]`` + header). + intent : str + ``'in'``, ``'out'``, or ``'inout'``. + is_optional : bool + active : str + Active condition in standard names (empty if always active). + active_local : str + Active condition translated to local Fortran names. + source : str + ``'host'``, ``'control'``, or ``'suite'``. + host_entry : HostVarEntry or None + The resolved host/control entry (``None`` for suite-owned vars + that have already been declared before this call). + suite_var : SuiteVar or None + The suite data entry (``None`` for host/control vars). + base_expr : str + Fortran access path (without dimension subscripts). + subscript : str + Dimension subscript string, e.g. ``'(lb:ub, 1:nlev)'`` or ``''``. + call_expr : str + Full call-site expression: ``base_expr + subscript``. + used_dim_std_names : set of str + Standard names of host/control/suite dimension variables + referenced in the subscript. Used to drive USE statements and + dummy-arg injection in the group cap. + used_const_dim_std_names : set of str + Standard names of *framework-constituent* dimension references + (notably ``number_of_ccpp_constituents``) referenced in the + subscript. These do not produce USE statements (the value is + reached via the per-instance constituent object), but they DO + appear in the host-facing introspection inputs list — original + capgen reports framework-constituent dim names there. + needs_unit_transform : bool + needs_kind_transform : bool + unit_forward : str + Fortran expression: host/suite → scheme (for pre-call, intent in/inout). + Empty if no transformation needed. + unit_backward : str + Fortran expression: scheme → host/suite (for post-call, intent out/inout). + Empty if no transformation needed. + kind_scheme : str + Kind declared in the scheme metadata. + kind_host : str + Kind of the host/suite variable. + temp_name : str + Name for the transformation temporary (``local_name + '_l'``). + ptr_name : str + Name for the optional pointer (``local_name + '_p'``). + transform_case : int + 1 = direct, 2 = pointer only, 3 = transform only, 4 = pointer+transform. + """ + standard_name: str + scheme_local_name: str + intent: str + is_optional: bool + active: str + active_local: str + source: str + host_entry: Optional[HostVarEntry] + suite_var: Optional['SuiteVar'] + base_expr: str + subscript: str + call_expr: str + used_dim_std_names: Set[str] + needs_unit_transform: bool + needs_kind_transform: bool + unit_forward: str + unit_backward: str + kind_scheme: str + kind_host: str + temp_name: str + ptr_name: str + transform_case: int + scheme_dimensions: List[str] + # ``needs_vert_flip`` is True when host and scheme metadata disagree on + # ``top_at_one``: the host-side access expression carries a reverse-stride + # subscript on the vertical axis and the transform pipeline copies through + # a temp local just like a unit conversion does. Composes with unit/kind + # transforms when present. + needs_vert_flip: bool = False + is_constituent_arg: bool = False + # ``is_constituent`` is True if the scheme metadata flagged this variable + # with any of ``constituent``, ``advected``, or ``molar_mass`` (a non-default + # value). Distinct from :attr:`is_constituent_arg`, which marks the + # ``ccpp_constituent_properties_t`` register-phase array argument. + is_constituent: bool = False + # For ``source == 'constituent'`` args: module that owns the constituent + # symbols this arg references (typically the suite cap module). ``None`` + # for non-constituent args. + constituent_module_name: Optional[str] = None + # Extra symbols (beyond :attr:`root_symbol`) that the group cap must USE + # from :attr:`constituent_module_name`. Typically ``index_of_`` + # integers referenced inside the constituent array subscript. + constituent_extra_symbols: Set[str] = field(default_factory=set) + # Framework-constituent dim std names referenced in the subscript + # (e.g. ``number_of_ccpp_constituents`` as the trailing axis of + # ``ccpp_constituents``). These are not USE'd from any module — the + # value is reached via the per-instance constituent object — but + # they're surfaced as inputs by the introspection routines in + # :mod:`generator.host_cap`. Replaces the older trick of stuffing + # them into :attr:`used_dim_std_names`. + used_const_dim_std_names: Set[str] = field(default_factory=set) + # Real (un-mangled) constituent base standard names that this arg's + # subscript indexes via ``index_of_``. The Fortran symbol in + # :attr:`constituent_extra_symbols` is mangled to fit the 63-char + # identifier limit (see :func:`_index_symbol_name`), so its suffix is + # NOT a reliable source of the real standard name. This set preserves + # the real names verbatim so ``ccpp_initialize_constituents`` can pass + # them to ``%const_index`` as the lookup key (and list them in + # ``ccpp_model_const_stdnames``). + constituent_index_std_names: Set[str] = field(default_factory=set) + + @property + def needs_transform(self) -> bool: + return (self.needs_unit_transform or self.needs_kind_transform + or self.needs_vert_flip) + + @property + def module_name(self) -> Optional[str]: + """Module to USE for this argument (``None`` for control vars).""" + if self.source == 'constituent': + return self.constituent_module_name + if self.host_entry is not None: + return self.host_entry.module_name + if self.suite_var is not None: + return self.suite_var.module_name + return None + + @property + def root_symbol(self) -> str: + """Root Fortran symbol name for the USE statement.""" + return _root_symbol(self.base_expr) + + +@dataclass +class ResolvedCall: + """All resolved arguments for one scheme phase call.""" + scheme_name: str + phase: str + args: List[ResolvedArg] = field(default_factory=list) + # Fortran module that exports the scheme's subroutines. Defaults to + # ``scheme_name`` for the common-case where the .meta table name and + # the Fortran module name match; overridden by the ``module_name`` + # attribute in ``[ccpp-table-properties]`` when the two differ. + scheme_module: str = '' + + @property + def used_modules(self) -> Dict[str, Set[str]]: + """Return ``{module_name: {symbol, ...}}`` for USE-statement building.""" + result: Dict[str, Set[str]] = {} + for arg in self.args: + mod = arg.module_name + if mod is not None: + sym = arg.root_symbol + result.setdefault(mod, set()).add(sym) + for dim_std in arg.used_dim_std_names: + pass # dim vars added separately by the group cap writer + return result + + +def _resolve_subcycle_loop_bound( + loop_str: Optional[str], + host_dict: Dict[str, HostVarEntry], + suite_vars: Optional[Dict[str, 'SuiteVar']] = None, +) -> Tuple[str, str]: + """Resolve a subcycle ``loop=`` attribute into Fortran source. + + Returns ``(fortran_expr, std_name)`` where: + + * *fortran_expr* is the value to splice into ``do ccpp_loop_counter + = 1, ``. For an absent or literal-integer value + this is the literal itself; for a CCPP standard name it is the + host's (or suite's) Fortran local name. + * *std_name* is the resolved CCPP standard name (lower-cased) when + the loop bound was a symbol, otherwise the empty string. Used by + the group cap to (a) emit ``use , only: `` for + host-owned bounds and (b) inject the bound as a dummy argument + when it's a control variable. + + Raises ``CCPPError`` if the loop bound is a non-integer token that + doesn't resolve against the host/control dictionary or the suite's + interstitial variables. + """ + if loop_str is None: + return '1', '' + raw = loop_str.strip() + if not raw: + return '1', '' + # Integer literal — pass through verbatim (also handles negative + # numbers for completeness, though those aren't physically meaningful). + try: + int(raw) + return raw, '' + except ValueError: + pass + # Treat as a CCPP standard name. Standard names are lower-cased at + # parse time; the XML attribute may carry a mixed-case spelling, so + # normalise before lookup. + key = raw.lower() + entry = host_dict.get(key) + if entry is not None: + # Use the full access_path — for a free module variable this is + # just the local name, but for a DDT-component the access path is + # ``%`` (or + # ``(instance_number)%`` when the parent is + # in an instance-dimensioned array; resolve that template here, + # and re-attach any literal local_subscript from a + # ``local_name = foo(1)``-style declaration). + return _render_value_expr(entry, host_dict), key + if suite_vars and key in suite_vars: + suite_var = suite_vars[key] + return suite_var.access_path, key + raise CCPPError( + "Subcycle loop=\"{}\" is not an integer literal and does not " + "resolve to a CCPP standard name in the host/control metadata or " + "as a suite-owned variable; declare it (typically in the " + "type=control or type=host table) before using it as a subcycle " + "loop bound".format(loop_str) + ) + + +@dataclass +class ResolvedSubcycle: + """A subcycle ``do`` loop wrapping one or more scheme run calls. + + Only appears in ``phase_calls['run']``; non-run phases are always flat + (subcycle boundaries are not meaningful for init/final). + + Attributes + ---------- + loop : str + Loop-count Fortran expression to splice into ``do + ccpp_loop_counter = 1, ``. An integer literal when the + XML attribute was a literal; otherwise the host's local Fortran + name resolved from the CCPP standard name in the XML. + loop_std_name : str + The resolved CCPP standard name (lower-cased) when *loop* came + from a symbol; empty string when *loop* is an integer literal. + Drives USE-statement emission and control-variable dummy-arg + injection in the group cap. + calls : list of :data:`PhaseItem` + Items wrapped by this loop. Element types: :class:`ResolvedCall` + for scheme calls; :class:`ResolvedSubcycle` for nested loops + (yes, this is recursive — SDFs may declare arbitrary subcycle + nesting and the resolver preserves that structure for the cap + emitter to render as nested ``do`` loops). + """ + loop: str + # Use a forward-ref string for ``PhaseItem`` because the alias is + # defined just below this class. Runtime type-checking still works. + calls: List['PhaseItem'] = field(default_factory=list) + loop_std_name: str = '' + + +# Type alias for the contents of a phase's call list (and a subcycle's +# inner items). PhaseItem is itself the union of plain scheme calls and +# nested subcycles, so a phase / subcycle can carry arbitrary nesting. +PhaseItem = Union[ResolvedCall, ResolvedSubcycle] + + +def iter_phase_calls(items: List[PhaseItem]): + """Yield every :class:`ResolvedCall` in *items*, recursing into + nested :class:`ResolvedSubcycle` items. Subcycles themselves are + not yielded — only the leaf scheme calls.""" + for item in items: + if isinstance(item, ResolvedCall): + yield item + elif isinstance(item, ResolvedSubcycle): + # Recurse so nested ``ResolvedSubcycle`` items unwrap too. + yield from iter_phase_calls(item.calls) + + +def iter_phase_subcycles(items: List[PhaseItem]): + """Yield every :class:`ResolvedSubcycle` in *items*, including nested + subcycles. Used by the group cap to emit nested ``do`` loops and + by ``_collect_host_io`` to find every loop bound.""" + for item in items: + if isinstance(item, ResolvedSubcycle): + yield item + yield from iter_phase_subcycles(item.calls) + + +@dataclass +class ResolvedGroup: + """Resolution results for one suite group.""" + group_name: str + # One list of PhaseItem objects per phase. Non-run phases contain only + # ResolvedCall; the run phase may also contain ResolvedSubcycle items. + phase_calls: Dict[str, List[PhaseItem]] = field(default_factory=dict) + # Module → symbols referenced by dimension lookups in this group. + dim_uses: Dict[str, Set[str]] = field(default_factory=dict) + + +# auto-clone-constituents: snapshot of one consumer-side +# ``is_constituent`` scheme arg whose ``std_name`` has no +# register-phase source. Carries everything the emitter needs to +# synthesise a ``%instantiate(...)`` call in ``_register``. +# Captured at resolve time so the emitter doesn't reach back into +# raw scheme metadata. +@dataclass +class AutoCloneEntry: + std_name: str + long_name: str + diag_name: str # diagnostic_name or local_name fallback + units: str + vertical_dim: str # standard name of the vertical axis + advected: bool + molar_mass: float + default_value: Optional[float] + min_value: Optional[float] + water_species: Optional[bool] + mixing_ratio_type: Optional[str] + + +@dataclass +class SuiteResolution: + """Complete resolution result for one suite. + + Attributes + ---------- + constituent_register_calls : list of (scheme_name, scheme_local_name) + For each register-phase scheme arg whose ``type`` is + ``ccpp_constituent_properties_t`` (intent=out, allocatable), records + the (scheme, scheme arg local name) pair. The suite cap uses this + list to emit two-pass merge logic that populates the host's + ``ccpp_model_constituents_object`` with the per-scheme constituent + arrays. Empty when no register-phase scheme produces constituents. + constituent_index_names : list of str + Sorted list of base-constituent standard names that need an + ``index_of_`` integer emitted in the suite cap. These are the + REAL (un-mangled) standard names, collected from every + ``source='constituent'`` ResolvedArg's + ``constituent_index_std_names`` set -- NOT recovered from the + (possibly mangled) ``index_of_*`` Fortran symbols. Used by + :mod:`generator.suite_cap` to emit the index declarations and the + ``ccpp_model_constituents_object%const_index`` population calls + in ``_init``. + uses_constituents : bool + True iff any scheme arg in this suite has ``source='constituent'`` + (excluding the legacy register-phase ``is_constituent_arg``). + Drives suite-cap emission of the ccpp_constituents / + ccpp_constituent_tendencies pointers and related state. + suite_init_call : ResolvedCall or None + Resolved call for the suite-level ```` scheme (if any). + The scheme's ``init`` phase is invoked once per ``_init`` + call (per instance, per suite), after the group ``state_alloc`` + loop and before the state transition to + ``CCPP_SUITE_FRAMEWORK_INITIALIZED``. + suite_final_call : ResolvedCall or None + Resolved call for the suite-level ```` scheme (if any). + The scheme's ``final`` phase is invoked once per ``_final`` + call (per instance, per suite), before the state transition to + ``CCPP_SUITE_UNREGISTERED``. + """ + suite_name: str + groups: List[ResolvedGroup] = field(default_factory=list) + suite_vars: Dict[str, SuiteVar] = field(default_factory=dict) + uses_instance_dimension: bool = False + constituent_register_calls: List[Tuple[str, str]] = field(default_factory=list) + constituent_index_names: List[str] = field(default_factory=list) + uses_constituents: bool = False + suite_init_call: Optional[ResolvedCall] = None + suite_final_call: Optional[ResolvedCall] = None + # auto-clone-constituents: populated only when the legacy shim is + # enabled. Each entry is one is_constituent consumer whose + # std_name has no register-phase source; the suite cap synthesises + # a ``%instantiate(...)`` call per entry into the per-suite + # dynamic-constituents buffer. Empty when the shim is off. + auto_cloned_constituents: List[AutoCloneEntry] = field(default_factory=list) + + # auto-clone-constituents: this property exists so consumer + # emitters (host_constituents, suite_cap) never have to read + # ``auto_cloned_constituents`` themselves. When the legacy + # auto-clone shim retires, drop the second clause below and the + # property collapses to ``bool(self.constituent_register_calls)``; + # every consumer keeps working without changes. + @property + def needs_dynamic_constituents_buffer(self) -> bool: + """True iff the per-suite ``_dynamic_constituents`` buffer + must be declared and populated for this suite. + + Single source of truth for the predicate. + """ + return bool( + self.constituent_register_calls + # auto-clone-constituents: + or self.auto_cloned_constituents + ) + + +######################################################################## +# Argument resolution helpers +######################################################################## + +def _local_name_conflict( + name: str, + existing_names: Set[str], +) -> str: + """Return *name* with a numeric suffix if it already exists in *existing_names*. + + Fortran identifiers are **case-insensitive**, so the collision check + is performed in lowercase: ``cp_l`` and ``CP_l`` are the same name + to the compiler and must not be emitted side-by-side as two locals. + The returned name preserves the input case (so generated source + keeps the metadata's spelling), but callers MUST add the lowercased + name to *existing_names* so subsequent calls see the collision. + """ + if name.lower() not in existing_names: + return name + # Split on last '_' to find the suffix ('_l' or '_p'). + if '_' in name: + base, suffix = name.rsplit('_', 1) + suffix = '_' + suffix + else: + base, suffix = name, '' + n = 2 + while True: + candidate = '{}_{}{}' .format(base, n, suffix) + if candidate.lower() not in existing_names: + return candidate + n += 1 + + +#: Standard names of the two loop-context control variables (see +#: doc/redesign_prompt.md §4.2). Scheme args declaring these resolve +#: to the generated do-loop locals emitted by the group cap — they are +#: in scope only inside a ```` block. +_LOOP_COUNTER_STD = 'ccpp_loop_counter' +_LOOP_EXTENT_STD = 'ccpp_loop_extent' + + +def _resolve_one_arg( + scheme_var, # MetaVar from scheme metadata + phase: str, + host_dict: Dict[str, HostVarEntry], + suite_vars: Dict[str, SuiteVar], + scheme_name: str, + used_local_names: Set[str], + suite_name: str = '', + loop_context: Optional[List[Tuple[str, Optional[str]]]] = None, + const_stds: Set[str] = frozenset(), +) -> ResolvedArg: + """Resolve one scheme argument against host/control/suite dictionaries. + + Implements the four variable-matching cases from Section 8.4. + + Parameters + ---------- + scheme_var : MetaVar + The variable entry from the scheme's phase metadata section. + phase : str + The current scheme phase (affects subscripting and error messages). + host_dict : dict + Flat host+control variable dict. + suite_vars : dict + Accumulated suite-owned variables (mutated if Case 2 applies). + scheme_name : str + Name of the enclosing scheme (for error messages). + used_local_names : set of str + Already-used local variable names in this group cap function (for + conflict resolution of temp/pointer names). + loop_context : list of (str, str or None), optional + Stack of ``(loop_count_expr, loop_std_name)`` pairs for the + enclosing ```` blocks, outermost first. Empty (or + omitted) when the call is not inside any subcycle. Used to + resolve ``ccpp_loop_counter`` / ``ccpp_loop_extent`` scheme + args against the generated do-loop locals. + + Returns + ------- + ResolvedArg + + Raises + ------ + CCPPError + Case 3: variable not found and intent is not ``'out'``. + """ + std_name = scheme_var.standard_name + intent = scheme_var.intent or 'in' + local = scheme_var.local_name + optional = scheme_var.optional + + # ---- loop-context std names (ccpp_loop_counter / ccpp_loop_extent) - + # Per design (doc/redesign_prompt.md §4.2): these are scoped to the + # body of a ````. Resolve them against the generated do- + # loop locals; outside a subcycle, raise a clear error pointing at + # the SDF contract rather than the host metadata. + if std_name in (_LOOP_COUNTER_STD, _LOOP_EXTENT_STD): + if not loop_context: + raise CCPPError( + "Scheme '{scheme}' (phase '{phase}') requests standard " + "name '{std}' for argument '{local}', but the scheme is " + "not placed inside a ```` block in the suite " + "definition file.\n" + "\n" + "'{std}' is a loop-context control variable scoped to a " + "subcycle do-loop body (see doc/redesign_prompt.md " + "§4.2). Either wrap the scheme in ``…`` in the SDF, or remove the " + "'{std}' argument from the scheme metadata.".format( + scheme=scheme_name, + phase=phase, + std=std_name, + local=local, + ) + ) + # Resolve to the OUTERMOST enclosing subcycle. Per the deferred + # item in doc/migration.md §8 ("Nested subcycle + # ccpp_loop_counter semantics"), nested-loop schemes that need + # the innermost counter aren't supported yet — every cam-sima + # / SCM use we've audited reads the OUTERMOST counter only. + outer_count_expr, _outer_std = loop_context[0] + if std_name == _LOOP_COUNTER_STD: + # The group cap emits the outermost do-loop with local + # variable ``ccpp_loop_counter`` (group_cap._loop_counter_name + # depth 1). Match that name verbatim — it's in scope wherever + # this scheme call site is emitted. + call_expr = 'ccpp_loop_counter' + else: # _LOOP_EXTENT_STD + # ``ccpp_loop_extent`` is the OUTERMOST subcycle's loop + # count — either an integer literal (e.g. ``'3'``) or a + # host-resolved local name (e.g. ``'n_sub'``) depending on + # how the SDF declared ``loop=``. + call_expr = outer_count_expr + return ResolvedArg( + standard_name=std_name, + scheme_local_name=local, + intent=intent, + is_optional=optional, + active='', + active_local='', + source='control', + host_entry=None, + suite_var=None, + base_expr=call_expr, + subscript='', + call_expr=call_expr, + used_dim_std_names=set(), + needs_unit_transform=False, + needs_kind_transform=False, + unit_forward='', + unit_backward='', + kind_scheme=scheme_var.kind, + kind_host='', + temp_name='', + ptr_name='', + transform_case=1, + scheme_dimensions=list(scheme_var.dimensions), + used_const_dim_std_names=set(), + ) + + # ---- detect constituent register args (special-cased) --------------- + # Schemes that register dynamic constituents declare an intent=out + # ``ccpp_constituent_properties_t`` allocatable array. Those arguments + # are NOT promoted to suite-owned data — they are local temporaries in + # the suite cap's ``_register`` subroutine, used to count and + # populate the host's ``ccpp_model_constituents_object`` via the + # two-pass merge pattern. + is_constituent = ( + phase == 'register' + and intent == 'out' + and scheme_var.type.strip() == _CONST_PROP_TYPE + ) + if is_constituent: + return ResolvedArg( + standard_name=std_name, + scheme_local_name=local, + intent=intent, + is_optional=optional, + active='', + active_local='', + source='constituent', + host_entry=None, + suite_var=None, + base_expr='scheme_consts', + subscript='', + call_expr='scheme_consts', + used_dim_std_names=set(), + needs_unit_transform=False, + needs_kind_transform=False, + unit_forward='', + unit_backward='', + kind_scheme=scheme_var.kind, + kind_host='', + temp_name='', + ptr_name='', + transform_case=1, + scheme_dimensions=list(scheme_var.dimensions), + is_constituent_arg=True, + ) + + # ---- detect constituent-sourced scheme args (framework auto-provision) + # Returns a synthesised ``source='constituent'`` ResolvedArg for: + # * scheme args declaring a framework-known std name + # (ccpp_constituents, ccpp_constituent_tendencies, + # number_of_ccpp_constituents, ccpp_constituent_properties); + # * scheme args flagged ``is_constituent`` with intent=in/inout + # (routed to ccpp_constituents(, index_of_)); + # * scheme args flagged ``is_constituent`` with intent=out and + # standard name starting with ``tendency_of_`` (routed to + # ccpp_constituent_tendencies(, index_of_)). + # Returns ``None`` if the arg is not constituent-related. + const_arg = _resolve_constituent_arg( + scheme_var, phase, host_dict, suite_vars, scheme_name, suite_name, + const_stds=const_stds, + ) + if const_arg is not None: + return const_arg + + # ---- determine source ----------------------------------------------- + host_entry: Optional[HostVarEntry] = host_dict.get(std_name) + + # active is a host-model-only attribute; read it from the host entry only. + # When the scheme arg is optional, the group cap emits the + # pointer-association pattern (transform_case 2 / 4) so the scheme + # sees PRESENT()=.false. when the active condition is false. When + # the scheme arg is non-optional, the group cap emits a runtime + # guard before the call: if (.not. (active)) raise errflg and return. + # The suite designer is responsible for ensuring the active condition + # holds at call time; the guard converts a silent invalid-memory read + # into a clear runtime error. + active = host_entry.active if host_entry is not None else '' + suite_var: Optional[SuiteVar] = suite_vars.get(std_name) + + if host_entry is not None and suite_var is None: + source = 'control' if host_entry.is_control else 'host' + elif suite_var is not None and host_entry is None: + source = 'suite' + elif host_entry is None and suite_var is None: + # Case 2 or 3. + if intent == 'out': + # This scheme is the first (in phase->scheme order) to provide + # this variable, so it DEFINES the suite-owned storage that the + # framework allocates in ``ccpp__data``. A character + # definer must specify a concrete length: ``len=*`` (assumed + # length) is only valid for a dummy argument, never for stored + # data, and a later ``len=*`` consumer/writer has no concrete + # length to inherit. Reject it here with a clear message rather + # than emitting an undeclarable ``character(len=*)`` component. + if ((scheme_var.type or '').strip().lower() == 'character' + and (scheme_var.kind or '').strip() == 'len=*'): + raise CCPPError( + "Suite-owned character variable '{}' (standard_name='{}') " + "is first defined as intent(out) by scheme '{}' (phase " + "'{}') with kind='len=*'; the defining scheme must declare " + "a concrete length (e.g. kind=len=512) because the " + "framework allocates storage for it in the suite data " + "module. Assumed length (len=*) is permitted only on " + "later schemes that consume or re-write the " + "variable.".format( + local, std_name, scheme_name, phase, + ) + ) + inst_entry = host_dict.get('instance_number') + inst_access = '({})'.format(inst_entry.local_name) if inst_entry else '(1)' + suite_var = SuiteVar( + standard_name=std_name, + local_name=_unique_suite_field(local, std_name, suite_vars), + type_=scheme_var.type, + kind=scheme_var.kind, + units=scheme_var.units, + dimensions=list(scheme_var.dimensions), + source_scheme=scheme_name, + source_phase=phase, + suite_module_name='ccpp_{}_data'.format(suite_name), + inst_access=inst_access, + allocatable=scheme_var.allocatable, + ) + suite_vars[std_name] = suite_var + source = 'suite' + else: + raise CCPPError( + "Variable '{}' (standard_name='{}') requested by scheme " + "'{}' phase '{}' with intent({}) is not provided by the host " + "metadata or by any prior scheme; " + "either add it to the host metadata or ensure an earlier " + "scheme provides it with intent(out)".format( + local, std_name, scheme_name, phase, intent + ) + ) + else: + # Both found — host takes precedence (suite data shouldn't duplicate host). + source = 'control' if host_entry.is_control else 'host' + + # ---- build access expression ----------------------------------------- + if host_entry is not None: + # ``host_entry.access_path`` is the verbatim form from + # build_flat_host_dict; for DDT-instance arrays it carries the + # ``(instance_number)`` template that needs codegen-time resolution. + base_expr = _substitute_instance_idx(host_entry.access_path, host_dict) + host_dims = host_entry.dimensions + host_units = host_entry.units + host_kind = host_entry.kind + host_type = host_entry.type + host_allocatable = host_entry.allocatable + else: + base_expr = suite_var.access_path + host_dims = suite_var.dimensions + host_units = suite_var.units + host_kind = suite_var.kind + host_type = suite_var.type_ + host_allocatable = suite_var.allocatable + + # ---- type identity check --------------------------------------------- + # The defining source (host metadata or the first scheme to write a + # suite-owned var) sets the variable's type; every subsequent consumer + # must agree. Numeric/kind coercion happens via the transform pipeline, + # but the *type kind* itself (real vs integer vs logical vs DDT) must + # match identically — there is no transform that crosses those. + if (host_type or '').strip().lower() != (scheme_var.type or '').strip().lower(): + raise CCPPError( + "Variable '{}' (standard_name='{}'): {} declares type='{}' but " + "scheme '{}' declares type='{}'; cross-type assignment is not " + "supported, the scheme metadata must match the defining type".format( + local, std_name, source, host_type, + scheme_name, scheme_var.type, + ) + ) + + # ---- rank check ------------------------------------------------------ + if len(host_dims) != len(scheme_var.dimensions): + raise CCPPError( + "Variable '{}' (standard_name='{}'): {} declares rank {} " + "(dimensions={}) but scheme '{}' declares rank {} " + "(dimensions={}); the scheme metadata's dimension list must " + "match the defining rank".format( + local, std_name, source, len(host_dims), + list(host_dims), scheme_name, len(scheme_var.dimensions), + list(scheme_var.dimensions), + ) + ) + + # ---- per-position dimension identity check --------------------------- + # Each dimension entry is canonicalized to ``lower:upper`` form (bare + # ``X`` -> ``ccpp_constant_one:X``) and compared for strict identity. + # No name aliasing happens here by default; the legacy-compat shim + # rewrites deprecated names at parse time, and the opt-in GFS + # dim-aliases shim (--gfs-dim-aliases) collapses a small audited + # list of physically-equivalent upper-bound names inside + # ``_canonical_dim`` itself. + for pos, (hdim, sdim) in enumerate(zip(host_dims, scheme_var.dimensions)): + if _canonical_dim(hdim) != _canonical_dim(sdim): + raise CCPPError( + "Variable '{}' (standard_name='{}'): {} declares " + "dimension {} as '{}' but scheme '{}' declares it as " + "'{}'; per-position dimension entries must match " + "(the scheme metadata's dimension list must name the " + "defining axes; bare names are equivalent to " + "ccpp_constant_one:, all other lower bounds are " + "distinct)".format( + local, std_name, source, pos, hdim, + scheme_name, sdim, + ) + ) + + # ---- allocatable compatibility check --------------------------------- + # An actual argument that is not allocatable cannot be passed to an + # allocatable dummy. The reverse direction (allocatable host -> plain + # assumed-shape dummy) is legal Fortran and is permitted: the scheme + # simply forgoes access to the allocation status. + if scheme_var.allocatable and not host_allocatable: + raise CCPPError( + "Variable '{}' (standard_name='{}'): scheme '{}' declares " + "allocatable=True but {} declares allocatable=False; " + "an allocatable dummy cannot receive a non-allocatable actual " + "argument".format( + local, std_name, scheme_name, source + ) + ) + + # ---- vertical-flip detection (top_at_one mismatch) ------------------ + # Both host and scheme declare a top_at_one attribute (default False). + # A mismatch triggers a reverse-stride substitution on the host-side + # subscript at the vertical-dim position so the array section is read + # (and written) in flipped order. Only meaningful when the variable + # actually has a vertical dimension. + if host_entry is not None: + host_top_at_one = host_entry.top_at_one + else: + host_top_at_one = False + scheme_top_at_one = bool(getattr(scheme_var, 'top_at_one', False)) + needs_vert_flip = ( + host_top_at_one != scheme_top_at_one + and any(_dim_has_vertical(d) for d in host_dims) + ) + + if host_allocatable: + # Allocatable actual arguments must omit explicit dimension ranges: + # the callee declares the dummy as allocatable too and assumes the + # array shape from the actual. + subscript: str = '' + used_dim_std: Set[str] = set() + else: + local_sub = host_entry.local_subscript if host_entry is not None else [] + subscript, used_dim_std = _build_merged_subscript( + host_dims, local_sub, phase, host_dict, suite_vars=suite_vars, + flip_vertical=needs_vert_flip, + ) + call_expr = base_expr + subscript + + # Scalar horizontal_dimension in a physics phase: the scheme is asking + # for the size of the horizontal slice it actually receives. During run + # the host passes a chunk (lb:ub); during the other physics phases the + # loop bounds collapse to 1:ncols. In both cases (ub - lb + 1) yields + # the correct extent, so we synthesise it from the loop-bound control + # variables and bypass the host's full-domain scalar (e.g. ncols). + if (phase in _PHYSICS_PHASES + and std_name == _HORIZ_DIM_STD + and not scheme_var.dimensions): + lb = host_dict.get(_HORIZ_BEGIN_STD) + ub = host_dict.get(_HORIZ_END_STD) + if lb is None or ub is None: + raise CCPPError( + "Scheme '{}' phase '{}' requests scalar '{}' but the host " + "metadata lacks '{}'/'{}' (required to compute the per-call " + "horizontal extent)".format( + scheme_name, phase, _HORIZ_DIM_STD, + _HORIZ_BEGIN_STD, _HORIZ_END_STD + ) + ) + call_expr = '({} - {} + 1)'.format(ub.local_name, lb.local_name) + used_dim_std.update({_HORIZ_BEGIN_STD, _HORIZ_END_STD}) + + # ---- active expression translation ----------------------------------- + active_local = _translate_active_expr(active, host_dict) + + # ---- transformation detection ---------------------------------------- + # Normalise both unit strings so that equivalent spellings (``m2`` and + # ``m+2``) compare equal and do not appear as bogus mismatches. + host_units = _normalize_unit_string(host_units) + scheme_units = _normalize_unit_string(scheme_var.units) + scheme_kind = scheme_var.kind + + fwd_fn = find_unit_conversion(host_units, scheme_units) if host_units != scheme_units else None + bwd_fn = find_unit_conversion(scheme_units, host_units) if host_units != scheme_units else None + + needs_unit = fwd_fn is not None or bwd_fn is not None + # Unit mismatch with no known conversion is an error only if units differ. + if host_units != scheme_units and not needs_unit: + raise CCPPError( + "Variable '{}' (standard_name='{}'): host units '{}' differ from " + "scheme '{}' units '{}' but no unit conversion is known; " + "add a conversion to metadata/unit_conversion.py or fix the " + "metadata".format( + local, std_name, host_units, scheme_name, scheme_units + ) + ) + + # Character kind handling (len=N / len=*): + # - len=* in the scheme is always compatible with any host len=. + # - Matching specific len=N values need no transform (naturally equal). + # - Mismatched specific lengths (len=N vs len=M) are a metadata error; + # the scheme must declare len=* or match the defining metadata exactly. + _host_is_len = host_kind.startswith('len=') if host_kind else False + _scheme_is_len = scheme_kind.startswith('len=') if scheme_kind else False + if _host_is_len or _scheme_is_len: + if scheme_kind != 'len=*' and host_kind != scheme_kind: + raise CCPPError( + "Character variable '{}' (standard_name='{}'): host declares " + "kind='{}' but scheme '{}' declares kind='{}'; scheme must " + "use kind=len=* or match the defining kind exactly".format( + local, std_name, host_kind, scheme_name, scheme_kind + ) + ) + needs_kind = False + else: + needs_kind = bool(host_kind) and bool(scheme_kind) and host_kind != scheme_kind + + # Forward transformation expression (host/suite → scheme local). + # ``call_expr`` already carries the flipped vertical subscript when + # ``needs_vert_flip`` is True, so the unit-conversion formula naturally + # composes the flip on the host-side RHS. For a pure-flip case (no + # unit conversion) we emit a plain copy ``temp = host(...flipped)``. + # For a pure-kind case (host.kind != scheme.kind, no unit conversion, + # no vertical flip), we emit an explicit ``TYPE(host, kind=K)`` cast + # so the temp is actually assigned; without this branch the temp was + # declared and the call site referenced it, but no assignment was + # emitted -- gfortran fell back to implicit typing and yielded a + # garbage / Inf value at runtime. + unit_forward = '' + if needs_unit and fwd_fn is not None and intent in ('in', 'inout'): + unit_forward = _apply_transform_formula(fwd_fn, call_expr, scheme_kind) + elif needs_kind and intent in ('in', 'inout'): + unit_forward = _kind_cast_expr( + scheme_var.type, call_expr, scheme_kind, + local=local, std_name=std_name, scheme_name=scheme_name, + ) + elif needs_vert_flip and not needs_unit and intent in ('in', 'inout'): + unit_forward = call_expr + + # Backward transformation expression (scheme local → host/suite). + unit_backward = '' + if needs_unit and bwd_fn is not None and intent in ('out', 'inout'): + unit_backward_expr = '{}_l'.format(local) + unit_backward = _apply_transform_formula(bwd_fn, unit_backward_expr, host_kind) + elif needs_kind and intent in ('out', 'inout'): + unit_backward = _kind_cast_expr( + scheme_var.type, '{}_l'.format(local), host_kind, + local=local, std_name=std_name, scheme_name=scheme_name, + ) + elif needs_vert_flip and not needs_unit and intent in ('out', 'inout'): + unit_backward = '{}_l'.format(local) + + needs_transform = needs_unit or needs_kind or needs_vert_flip + + # ---- local variable names (transformation temp + pointer) ------------ + # ``used_local_names`` stores the LOWERCASED names so collision + # detection is Fortran-case-insensitive (see _local_name_conflict). + temp_name = '' + ptr_name = '' + if needs_transform: + candidate = '{}_l'.format(local) + temp_name = _local_name_conflict(candidate, used_local_names) + used_local_names.add(temp_name.lower()) + + if optional: + candidate = '{}_p'.format(local) + ptr_name = _local_name_conflict(candidate, used_local_names) + used_local_names.add(ptr_name.lower()) + + # ---- transform case -------------------------------------------------- + if optional and needs_transform: + transform_case = 4 + elif optional: + transform_case = 2 + elif needs_transform: + transform_case = 3 + else: + transform_case = 1 + + return ResolvedArg( + standard_name=std_name, + scheme_local_name=local, + intent=intent, + is_optional=optional, + active=active, + active_local=active_local, + source=source, + host_entry=host_entry, + suite_var=suite_var if source == 'suite' else None, + base_expr=base_expr, + subscript=subscript, + call_expr=call_expr, + used_dim_std_names=used_dim_std, + needs_unit_transform=needs_unit, + needs_kind_transform=needs_kind, + unit_forward=unit_forward, + unit_backward=unit_backward, + kind_scheme=scheme_kind, + kind_host=host_kind, + temp_name=temp_name, + ptr_name=ptr_name, + transform_case=transform_case, + scheme_dimensions=list(scheme_var.dimensions), + needs_vert_flip=needs_vert_flip, + is_constituent=scheme_var.is_constituent, + ) + + +######################################################################## +# Constituent-source synthesis (framework auto-provisioning) +######################################################################## + +def _const_dim_part( + dim: str, + phase: str, + host_dict: Dict[str, HostVarEntry], + suite_vars: Optional[Dict[str, 'SuiteVar']] = None, +) -> Tuple[str, Set[str], Set[str], Set[str]]: + """One-dim subscript with framework-constituent dim recognition. + + Returns ``(part, used_host_std, used_const_std, used_const_dim_std)``. + + The trailing dim ``number_of_ccpp_constituents`` is emitted as + ``':'`` (whole-axis slice). The std name is added to + ``used_const_dim_std`` so the introspection routine + (:func:`generator.host_cap._collect_host_io`) can include it in + its inputs list — original capgen reports framework-constituent dim + names there. No USE statement is emitted for the name: it isn't in + host_dict (the framework provides it via the per-instance + constituent object), so ``_collect_group_uses`` and + ``_extra_dim_ctrl_entries`` both silently skip it. All other dims + fall through to :func:`_one_dim_part` and their std names go into + ``used_host_std``. + + ``used_const_std`` collects framework-constituent *symbols* that + need a USE statement (currently unused at this layer; reserved for + future framework-constituent symbols that might appear inside a + dim expression). + """ + if dim == _CONST_NUM_STD or dim.endswith(':' + _CONST_NUM_STD): + return ':', set(), set(), {_CONST_NUM_STD} + part, used = _one_dim_part(dim, phase, host_dict, suite_vars=suite_vars) + return part, used, set(), set() + + +def _build_const_subscript( + dimensions: List[str], + phase: str, + host_dict: Dict[str, HostVarEntry], + suite_vars: Optional[Dict[str, 'SuiteVar']] = None, +) -> Tuple[str, Set[str], Set[str], Set[str]]: + """Build a subscript for a constituent-sourced arg. + + Like :func:`_build_call_subscript` but recognises + ``number_of_ccpp_constituents`` as a whole-axis slice. Returns + ``(subscript, used_host_std, used_const_std, used_const_dim_std)``; + *used_const_std* collects framework-constituent symbols that need + a USE statement (reserved for future use), and + *used_const_dim_std* collects framework-constituent dim std names + (e.g. ``number_of_ccpp_constituents``) for introspection. + """ + if not dimensions: + return '', set(), set(), set() + parts: List[str] = [] + used_host: Set[str] = set() + used_const: Set[str] = set() + used_const_dim: Set[str] = set() + for dim in dimensions: + part, uh, uc, ucd = _const_dim_part(dim, phase, host_dict, suite_vars) + parts.append(part) + used_host.update(uh) + used_const.update(uc) + used_const_dim.update(ucd) + return ('({})'.format(', '.join(parts)), + used_host, used_const, used_const_dim) + + +def _resolve_constituent_arg( + scheme_var, + phase: str, + host_dict: Dict[str, HostVarEntry], + suite_vars: Dict[str, 'SuiteVar'], + scheme_name: str, + suite_name: str, + const_stds: Set[str] = frozenset(), +) -> Optional[ResolvedArg]: + """Synthesise a ``source='constituent'`` ResolvedArg, or return ``None``. + + Per-instance access pattern: every constituent state lookup goes + through ``ccpp_model_constituents_obj()%`` where + *inst_num* is the host's local name for ``instance_number`` (or + ``1`` if the host doesn't declare it). The ``index_of_`` + integers and ``ccpp_model_const_stdnames`` parameter array are + module-level scalars on ``ccpp_host_constituents`` (identical + across instances). + + Host metadata always wins: if the host declares ``std_name`` as a + regular variable, this routine returns ``None`` and normal host-arg + resolution takes over. Constituent auto-provisioning is reserved + for names the host has not claimed. + + Three argument categories are recognised: + + 1. **Framework-named std_name** — one of + :data:`_FRAMEWORK_CONST_STDS` or starts with ``index_of_`` *and* + not declared by the host. + + * ``ccpp_constituents`` → ``ccpp_model_constituents_obj(inst)%vars_layer`` + * ``ccpp_constituent_tendencies`` → ``...%vars_layer_tend`` + * ``ccpp_constituent_properties`` → ``...%const_metadata`` + * ``number_of_ccpp_constituents`` → ``...%num_layer_vars`` (scalar) + * ``index_of_`` → ``index_of_`` (module-level integer) + + 2. **Base constituent** — ``scheme_var.is_constituent`` true, intent + in/inout, std_name not a ``tendency_of_*``. Routed to + ``...%vars_layer(, index_of_)``. + + 3. **Constituent tendency** — ``scheme_var.is_constituent`` true, + intent=out, std_name=``tendency_of_``. Routed to + ``...%vars_layer_tend(, index_of_)``. + + Mismatched combinations are hard errors (see error messages below). + The constituent arg always carries ``instance_number`` in + ``used_dim_std_names`` (when the host declares it) so the group cap + auto-injects it as a dummy via :func:`_extra_dim_ctrl_entries`. + """ + std_name = scheme_var.standard_name + intent = scheme_var.intent or 'in' + local = scheme_var.local_name + optional = scheme_var.optional + scheme_dims = list(scheme_var.dimensions) + + is_tendency_name = std_name.startswith(_TEND_PREFIX) + is_index_name = std_name.startswith(_INDEX_PREFIX) + is_framework_name = std_name in _FRAMEWORK_CONST_STDS or is_index_name + + # Rule (b): an UNFLAGGED consumer of a name that some scheme declares as a + # constituent -- a base constituent (``advected``, read via vars_layer) or a + # constituent tendency (``constituent`` on a ``tendency_of_*`` producer, read + # via vars_layer_tend) -- resolves to the SAME framework column the + # producer/registration backs. Consumers must not re-flag it: whether a + # given standard name is a constituent or an ordinary variable is the host's + # decision (CAM-SIMA vs CCPP-SCM), so we infer it from the + # scheme-metadata-wide set instead of the consumer's own metadata. + # Host/earlier-suite provision WINS: if this host declares the name, or a + # prior scheme already produced it as an ordinary variable, defer to normal + # resolution (a genuine constituent is in neither host_dict nor suite_vars). + is_known_constituent = std_name in const_stds + inferred_constituent_consumer = ( + is_known_constituent + and not scheme_var.is_constituent + and intent in ('in', 'inout') + and not (host_dict and std_name in host_dict) + and std_name not in suite_vars + ) + + # Host/suite provides it -> not a framework auto-provision. Defer to + # normal host/suite resolution when: + # + # * the host declares this std_name as a regular variable (e.g. a + # `protected integer` named `ntcw` with standard_name = + # index_of_..._tracer_concentration_array), OR + # * an earlier-phase scheme already produced it (it is in suite_vars). + # ``suite_vars`` accumulates across phases in chronological order + # (register, init, timestep_init, run, ...), so e.g. a band index + # produced by ``rrtmgp_inputs_setup_init`` (intent=out) is visible + # here when ``rrtmgp_sw_cloud_optics_run`` consumes it (intent=in). + # + # Constituent auto-provisioning is reserved for framework-named + # std_names that nothing else provides. + if is_framework_name and ( + (host_dict and std_name in host_dict) or std_name in suite_vars + ): + return None + + # A scheme that OUTPUTS ``index_of_`` is producing an ordinary index + # variable (e.g. ``rrtmgp_inputs_setup`` computing the diagnostic + # shortwave band index), NOT a constituent index -- constituent indices + # are read-only module integers bound by ``%const_index`` and are never + # written by a scheme. Defer so it becomes a suite var that later-phase + # consumers resolve via the gate above. (Index names produced by some + # OTHER scheme but consumed here have already been caught by the + # suite_vars branch above; this handles the producing arg itself, whose + # name is not yet in suite_vars on first occurrence.) + if is_index_name and intent == 'out': + return None + + constituent_module = _constituent_module_name(suite_name) + inst_entry = host_dict.get(_INSTANCE_NUM_STD) if host_dict else None + inst_local = inst_entry.local_name if inst_entry else None + inst_idx = inst_local if inst_local else '1' + + def _common_kwargs(base_expr, subscript, call_expr, + used_host_std, extra_symbols, + used_const_dim_std=None, + index_std_names=None): + used_host_std = set(used_host_std) + if inst_local: + used_host_std.add(_INSTANCE_NUM_STD) + return dict( + standard_name=std_name, + scheme_local_name=local, + intent=intent, + is_optional=optional, + active='', + active_local='', + source='constituent', + host_entry=None, + suite_var=None, + base_expr=base_expr, + subscript=subscript, + call_expr=call_expr, + used_dim_std_names=used_host_std, + needs_unit_transform=False, + needs_kind_transform=False, + unit_forward='', + unit_backward='', + kind_scheme=scheme_var.kind, + kind_host='', + temp_name='', + ptr_name='', + transform_case=1, + scheme_dimensions=scheme_dims, + is_constituent=scheme_var.is_constituent, + constituent_module_name=constituent_module, + constituent_extra_symbols=extra_symbols, + used_const_dim_std_names=(set(used_const_dim_std) + if used_const_dim_std else set()), + constituent_index_std_names=(set(index_std_names) + if index_std_names else set()), + ) + + # ---- Path 1a: index_of_ — module-level integer, no per-instance -- + if is_index_name: + # Mangle long std_names down to a Fortran-legal 63-char symbol; + # identity for short names, so existing fixtures are unaffected. + # ``_INDEX_PREFIX`` is already part of std_name -- strip then + # re-add via the helper for uniform truncation. + index_base = std_name[len(_INDEX_PREFIX):] + index_sym = _index_symbol_name(index_base) + return ResolvedArg(**_common_kwargs( + base_expr=index_sym, subscript='', call_expr=index_sym, + used_host_std=set(), extra_symbols={index_sym}, + index_std_names={index_base}, + )) + + # ---- Path 1b: framework-named std_name → DDT member ----------------- + if is_framework_name: + member = _FRAMEWORK_NAME_TO_MEMBER[std_name] + subscript, used_host_std, used_const_std, used_const_dim_std = \ + _build_const_subscript( + scheme_dims, phase, host_dict, suite_vars, + ) + base_expr = '{}({})%{}'.format(_CONST_OBJ_VAR, inst_idx, member) + call_expr = base_expr + subscript if subscript else base_expr + # Framework-constituent dim refs (e.g. number_of_ccpp_constituents) + # travel on the dedicated used_const_dim_std_names channel — no + # USE statement, but surfaced as inputs by the introspection + # routines in generator.host_cap. + return ResolvedArg(**_common_kwargs( + base_expr=base_expr, subscript=subscript, call_expr=call_expr, + used_host_std=used_host_std, + extra_symbols={_CONST_OBJ_VAR} | used_const_std, + used_const_dim_std=used_const_dim_std, + )) + + if not (scheme_var.is_constituent or inferred_constituent_consumer): + return None # not constituent-related + + # ---- Provider gate (mirrors original-capgen ConstituentVarDict.find_variable) + # A constituent-FLAGGED consumer whose standard name is actually + # PROVIDED elsewhere is an ordinary interstitial, not a constituent: + # + # * the host declares it in metadata -> host scope, or + # * an earlier scheme produced it intent=out -> already recorded + # in ``suite_vars`` (the resolver walks calls in execution order, + # so a producer that runs before this consumer is visible here). + # + # Original capgen only auto-creates a constituent when its + # ``find_variable`` returns None (nothing in host or suite scope + # provides the name). Without this gate a dry mixing ratio that is + # PRODUCED by e.g. ``wet_to_dry_water_vapor`` (intent=out, unflagged) + # but CONSUMED by ``kessler`` (advected=True) gets split into a suite + # array (producer) and a constituent column (consumer) AND spuriously + # auto-registered as a constituent. Defer to normal host/suite + # dispatch so producer and consumer share one storage location and the + # name is never registered as a constituent. Tendencies (intent=out, + # ``tendency_of_*``) are genuine constituent-tendency OUTPUTS and are + # never gated. + if not is_tendency_name and ( + std_name in suite_vars or std_name in host_dict + ): + return None + + # ---- Paths 2/3: is_constituent base or tendency --------------------- + if intent == 'out': + if not is_tendency_name: + raise CCPPError( + "Constituent-flagged scheme arg '{}' (standard_name='{}', " + "scheme='{}', phase='{}') has intent=out but its standard " + "name does not start with 'tendency_of_'. Physics phases " + "may only produce constituent tendencies; new base " + "constituents must be declared via a " + "ccpp_constituent_properties_t argument in a register-phase " + "scheme.".format(local, std_name, scheme_name, phase) + ) + base_std = std_name[len(_TEND_PREFIX):] + member = 'vars_layer_tend' + else: # in / inout + if is_tendency_name: + # Consumer of a constituent tendency (rule b): read the SAME column + # the producer wrote -- ccpp_constituent_tendencies(, + # index_of_). Reaching here means it is a recognised + # constituent tendency (flagged, or inferred via const_stds) + # that neither the host nor an earlier suite var provides. + base_std = std_name[len(_TEND_PREFIX):] + member = 'vars_layer_tend' + else: + base_std = std_name + member = 'vars_layer' + + leading_sub, used_host_std = _build_call_subscript( + scheme_dims, phase, host_dict, suite_vars=suite_vars, + ) + index_sym = _index_symbol_name(base_std) + if leading_sub: + subscript = leading_sub[:-1] + ', ' + index_sym + ')' + else: + subscript = '(' + index_sym + ')' + base_expr = '{}({})%{}'.format(_CONST_OBJ_VAR, inst_idx, member) + call_expr = base_expr + subscript + + return ResolvedArg(**_common_kwargs( + base_expr=base_expr, subscript=subscript, call_expr=call_expr, + used_host_std=used_host_std, + extra_symbols={index_sym, _CONST_OBJ_VAR}, + index_std_names={base_std}, + )) + + +######################################################################## +# Suite resolution +######################################################################## + +def resolve_suite( + suite, # generator.suite_xml.Suite + scheme_store, # metadata.variable_resolver.SchemeStore + host_dict: Dict[str, HostVarEntry], + phases: Optional[List[str]] = None, +) -> SuiteResolution: + """Resolve all scheme arguments for every group and phase in *suite*. + + Parameters + ---------- + suite : Suite + Parsed suite XML object. + scheme_store : SchemeStore + Scheme metadata organised for lookup. + host_dict : dict + Flat host+control variable dictionary. + phases : list of str, optional + Phases to resolve. Defaults to all six phases in chronological order + (register first), so that suite-owned variables produced by + ``_register`` are visible as dimensions or as ``intent(in)`` reads + in subsequent phases. + + Returns + ------- + SuiteResolution + + Raises + ------ + CCPPError + On any variable matching failure. + """ + if phases is None: + phases = ['register', 'init', 'timestep_init', 'run', + 'timestep_final', 'final'] + + # Validate up-front: every scheme name referenced by this suite — + # in any group (including nested subcycles/subcols) AND the + # suite-level / hooks — MUST be present in the scheme + # store. When a scheme is missing the resolver silently emits + # empty phase entries, which the cap generator then writes as a + # syntactically valid but semantically empty group cap (the user + # gets a successful build with the wrong runtime behaviour). + # Surface the configuration error here with the full list of + # missing schemes and a remediation pointer. + referenced_schemes: List[str] = list(suite.all_scheme_names()) + if suite.init_scheme: + referenced_schemes.append(suite.init_scheme) + if suite.final_scheme: + referenced_schemes.append(suite.final_scheme) + missing: List[str] = [] + seen_missing: Set[str] = set() + for sname in referenced_schemes: + if not scheme_store.has_scheme(sname) and sname not in seen_missing: + missing.append(sname) + seen_missing.add(sname) + if missing: + raise CCPPError( + "Suite '{suite}' references {n} scheme(s) whose metadata is " + "not loaded:\n\n" + " {names}\n\n" + "These schemes are listed in the SDF but no matching " + "``[ccpp-table-properties] type = scheme`` table is available " + "in the metadata files passed via ``--scheme-files``. Add " + "the missing scheme ``.meta`` files to the generator's " + "--scheme-files argument.".format( + suite=suite.name, + n=len(missing), + names='\n '.join(missing), + ) + ) + + # Detect whether any host variable uses the instance dimension + # specifically (multi-instance API marker on SuiteResolution). This + # is narrower than the general "registered scalar-index dim" check — + # we want to know only about the multi-instance pair here, not + # number_of_threads or future additions. + uses_instance = any( + 'number_of_instances' in entry.dimensions + for entry in host_dict.values() + ) + + suite_vars: Dict[str, SuiteVar] = {} + # Pre-create one ResolvedGroup per SDF group so the phase-major loop below + # can append each phase's calls to the right group. + resolved_groups: List[ResolvedGroup] = [ + ResolvedGroup(group_name=group.name) for group in suite.groups + ] + + # Resolve PHASE-MAJOR, GROUP-MINOR (groups in SDF order within each phase), + # mirroring the runtime execution hierarchy: the host completes EVERY + # group's ``init`` before any group's ``run``, every group's + # ``timestep_init`` before any ``run``, and so on. ``suite_vars`` + # accumulates in that true execution order, so a variable produced in an + # earlier phase by ANY group is visible to a consumer in a later phase of + # ANY group (cross-group cross-phase provision). Within a single phase, + # groups resolve in SDF order, so an earlier group may provide to a later + # one (e.g. ``physics_before_coupler`` -> ``physics_after_coupler``, which + # the host runs sequentially with coupling in between); a same-phase + # consumer that precedes its producer is still correctly rejected. + # + # (The previous nesting was group-major — each group through all its + # phases — which made a variable produced by a later group's ``init`` + # invisible to an earlier group's ``run`` even though, at runtime, all + # inits precede all runs.) + for phase in phases: + for group, resolved_group in zip(suite.groups, resolved_groups): + used_local_names_phase: Set[str] = set() + + if phase == 'run': + # Preserve subcycle structure for run-phase loop generation. + items_for_phase = _resolve_run_phase( + group, phase, scheme_store, host_dict, suite_vars, + used_local_names_phase, + suite_name=suite.name, + ) + else: + # Non-run phases: flatten all subcycles and silently + # deduplicate scheme names within the group. A scheme that + # appears multiple times in the suite XML (typically because + # it runs once per constituent in the ``run`` phase) must + # still have its register/init/finalize entry points invoked + # exactly once per group — matches ``design_init_dedup.md``. + scheme_names_flat = _dedup_scheme_names( + _collect_scheme_names(group) + ) + items_for_phase = _resolve_flat_phase( + scheme_names_flat, phase, scheme_store, host_dict, + suite_vars, used_local_names_phase, + suite_name=suite.name, + ) + + if items_for_phase: + resolved_group.phase_calls[phase] = items_for_phase + + # Collect dimension USE info once every phase/group is resolved, so + # ``suite_vars`` is complete (a dimension may reference a suite var + # produced by any group in any phase). + for resolved_group in resolved_groups: + resolved_group.dim_uses = _collect_dim_uses( + resolved_group, host_dict, suite_vars=suite_vars, + ) + + # Constituent register calls: gather the (scheme_name, scheme_local_name) + # pairs for every register-phase arg that was flagged as a constituent. + # The suite cap uses these to emit two-pass merge logic. + constituent_calls: List[Tuple[str, str]] = [] + for resolved_group in resolved_groups: + for resolved_call in iter_phase_calls(resolved_group.phase_calls.get('register', [])): + for arg in resolved_call.args: + if arg.is_constituent_arg: + constituent_calls.append( + (resolved_call.scheme_name, arg.scheme_local_name) + ) + # Walk every constituent-sourced arg (excluding the legacy + # register-phase ccpp_constituent_properties_t case) and collect: + # * uses_constituents — whether any constituent state is referenced + # * constituent_index_names — base std names X needing an index_of_X + uses_constituents = False + index_names: Set[str] = set() + for resolved_group in resolved_groups: + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: + if arg.source != 'constituent' or arg.is_constituent_arg: + continue + uses_constituents = True + # Collect the REAL (un-mangled) base standard names, not + # the mangled symbol suffix. ``ccpp_initialize_constituents`` + # passes these verbatim to ``%const_index``; a mangled key + # would never match a registered constituent and leave the + # ``index_of_`` integer at its 0 default -> out-of-bounds + # subscript at run time. + index_names.update(arg.constituent_index_std_names) + constituent_index_names = sorted(index_names) + + # Under option A the constituent object is generator-owned (lives in + # the ccpp_host_constituents module), so the host is no longer + # required to declare ``ccpp_model_constituents_object`` in its + # type=host metadata. No validation is needed here. + + # ---- suite-level / schemes ------------------------------ + # SDF v2.0 schema accepts an optional single ```` and ```` + # scheme name at the suite root. Resolve each to a ResolvedCall against + # the scheme's ``init`` / ``final`` phase metadata respectively. The + # local-name dedup set is fresh per call (these calls live outside any + # group and don't share locals with group phases). + suite_init_call: Optional[ResolvedCall] = None + suite_final_call: Optional[ResolvedCall] = None + if suite.init_scheme: + suite_init_locals: Set[str] = set() + suite_init_call = _resolve_one_call( + suite.init_scheme, 'init', scheme_store, host_dict, + suite_vars, suite_init_locals, suite_name=suite.name, + ) + if suite_init_call is None: + raise CCPPError( + "Suite '{}' declares {} but scheme '{}' " + "has no ``init`` phase in its metadata.".format( + suite.name, suite.init_scheme, suite.init_scheme, + ) + ) + if suite.final_scheme: + suite_final_locals: Set[str] = set() + suite_final_call = _resolve_one_call( + suite.final_scheme, 'final', scheme_store, host_dict, + suite_vars, suite_final_locals, suite_name=suite.name, + ) + if suite_final_call is None: + raise CCPPError( + "Suite '{}' declares {} but scheme '{}' " + "has no ``final`` phase in its metadata.".format( + suite.name, suite.final_scheme, suite.final_scheme, + ) + ) + + # auto-clone-constituents: collect synthesised %instantiate + # snapshots when the legacy shim is enabled. Returns [] when + # disabled (no-op for default builds). + auto_cloned = _collect_auto_clone_entries(resolved_groups, scheme_store) + + return SuiteResolution( + suite_name=suite.name, + groups=resolved_groups, + suite_vars=suite_vars, + constituent_register_calls=constituent_calls, + uses_instance_dimension=uses_instance, + constituent_index_names=constituent_index_names, + uses_constituents=uses_constituents, + suite_init_call=suite_init_call, + suite_final_call=suite_final_call, + # auto-clone-constituents: empty in default builds. + auto_cloned_constituents=auto_cloned, + ) + + +def validate_init_dimensions(suite_res: SuiteResolution) -> None: + """Reject suite-owned vars whose suite-time allocation can't be sized. + + capgen allocates every non-allocatable, dimensioned suite-owned + variable once in ``suite_data_init_fields``, which runs at the very + start of ``_init`` -- before any ``init`` / ``timestep_init`` / + ``run`` scheme code. Only the ``register`` phase completes earlier, so + a dimension whose value is written by a scheme in any later phase is not + yet set when the allocation happens (the size would be uninitialised + memory). Such a variable must instead be declared ``allocatable`` + (Fortran ``allocatable, intent(out)`` + metadata ``allocatable = True``) + so its producing scheme allocates it once the size is known. + + This is sound (no false positives) but intentionally partial: it sees + only dimensions a *scheme* writes. A host that recomputes a host-owned + dimension in its own driver each step is invisible here -- there is no + metadata signal for it. + + Raises + ------ + CCPPError + If a non-allocatable suite-owned variable is dimensioned by a + standard name written by a scheme in a phase after ``register``. + """ + written_after_register: Set[str] = set() + for resolved_group in suite_res.groups: + for phase, items in resolved_group.phase_calls.items(): + if phase == 'register': + continue + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: + if (arg.intent or 'in') in ('out', 'inout'): + written_after_register.add(arg.standard_name) + + for suite_var in suite_res.suite_vars.values(): + if suite_var.allocatable or not suite_var.dimensions: + continue + for dim in suite_var.dimensions: + for token in dim.split(':'): + token = token.strip() + if token in written_after_register: + raise CCPPError( + "Suite-owned variable '{var}' is allocated by the " + "suite at init time (in suite_data_init_fields), but " + "its dimension '{dim}' is written by a scheme in a " + "phase after 'register', so its size is not yet known " + "when the allocation runs (the allocation would use " + "uninitialised memory).\n" + " Declare '{var}' allocatable -- 'allocatable, " + "intent(out)' in the producing scheme's Fortran and " + "'allocatable = True' in its metadata -- so the scheme " + "allocates it once '{dim}' is set.".format( + var=suite_var.standard_name, dim=token, + ) + ) + + +# auto-clone-constituents: BEGIN legacy-shim helpers. Delete this +# block together with the rest of the auto-clone-constituents +# touchpoints; nothing else in the resolver references these. + +def _vertical_dim_of(scheme_var) -> str: + """Extract the vertical-axis standard name from a constituent + consumer's ``dimensions`` list. + + Returns the upper-bound std name of the first vertical-axis + dimension entry (matches :data:`_VDIM_STDS`). Falls back to + ``'vertical_layer_dimension'`` when the scheme arg carries no + vertical dim (e.g. a 1-D horizontal-only consumer) — that's the + framework default and matches original capgen's behaviour. + """ + for dim in getattr(scheme_var, 'dimensions', ()) or (): + upper = dim.split(':', 1)[-1].strip().lower() if ':' in dim else dim.strip().lower() + if upper in _VDIM_STDS: + return upper + return 'vertical_layer_dimension' + + +def _synthesised_long_name_from_std(std_name: str) -> str: + """auto-clone-constituents: fall back to a human-readable long_name + derived from the std_name when the metadata doesn't supply one. + + Mirrors original capgen's auto-clone behaviour: replace each + underscore with a space, then capitalise the first character. + ``cloud_liquid_dry_mixing_ratio`` → ``Cloud liquid dry mixing ratio``. + + Keeps the auto-clone shim's emitted ``%instantiate(long_name=...)`` + consistent with what original capgen produces so existing legacy + fixtures (e.g. CAM-SIMA's advection_test) don't need a metadata + edit just for the long_name property. + """ + return std_name.replace('_', ' ').capitalize() + + +def _make_auto_clone_entry(scheme_var) -> 'AutoCloneEntry': + """Snapshot one scheme MetaVar into an :class:`AutoCloneEntry`. + + Captures every field the emitter needs to synthesise a + ``%instantiate(...)`` call: the required kwargs (std_name, + long_name, diag_name, units, vertical_dim) and the optional + kwargs (advected, molar_mass, default_value, min_value, + water_species, mixing_ratio_type) with ``None`` for any optional + the metadata didn't set so the emitter can omit it. + """ + # ``diagnostic_name`` is a property that falls back to local_name + # when neither diagnostic_name nor diagnostic_name_fixed was set. + diag = scheme_var.diagnostic_name or scheme_var.local_name + # auto-clone-constituents: when the scheme metadata omits the + # long_name attribute, synthesise one from the std_name (matches + # original capgen's behaviour — see ``_synthesised_long_name_from_std``). + long_name = ( + scheme_var.long_name + or _synthesised_long_name_from_std(scheme_var.standard_name) + ) + return AutoCloneEntry( + std_name=scheme_var.standard_name, + long_name=long_name, + diag_name=diag, + units=scheme_var.units, + vertical_dim=_vertical_dim_of(scheme_var), + advected=bool(scheme_var.advected), + molar_mass=float(scheme_var.molar_mass or 0.0), + default_value=scheme_var.default_value, + min_value=scheme_var.min_value, + water_species=scheme_var.water_species, + mixing_ratio_type=scheme_var.mixing_ratio_type, + ) + + +def _lookup_scheme_var(scheme_store, scheme_name, phase, scheme_local_name): + """Return the scheme MetaVar matching *scheme_local_name* in + *scheme_name*'s *phase*, or ``None`` if not found. + + Used by the auto-clone collector to recover the full scheme + MetaVar (which carries the constituent-property fields) from a + ResolvedArg, which only carries a slim subset. + """ + vars_list = scheme_store.variables_for(scheme_name, phase) + if not vars_list: + return None + for mv in vars_list: + if mv.local_name == scheme_local_name: + return mv + return None + + +def _collect_auto_clone_entries(resolved_groups, scheme_store): + """Walk every ``is_constituent`` consumer arg and produce one + :class:`AutoCloneEntry` per unique standard name. + + Skips: + + * register-phase ``ccpp_constituent_properties_t`` args + (``is_constituent_arg`` is True for those — the scheme handles + its own registration explicitly); + * ``tendency_of_*`` std names (tendencies consume a constituent + but are not themselves a constituent registration); + * framework-named std names (``ccpp_constituents``, + ``ccpp_constituent_tendencies``, ``ccpp_constituent_properties``, + ``number_of_ccpp_constituents``, and ``index_of_*``) — these + reference the framework-provided constituent ARRAYS or scalar + counts, not individual species, and are not registrations; + * duplicates within the suite (first-occurrence wins; the + framework's runtime ``is_match`` would dedupe across the + eventual ``%new_field`` calls anyway). + + No-op when the auto-clone shim is disabled — returns an empty + list so :class:`SuiteResolution.auto_cloned_constituents` stays + empty in default builds. + """ + if not auto_clone_constituents.is_enabled(): + return [] + out: List['AutoCloneEntry'] = [] + seen: Set[str] = set() + for resolved_group in resolved_groups: + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: + if arg.source != 'constituent' or arg.is_constituent_arg: + continue + std = (arg.standard_name or '').strip().lower() + if not std or std.startswith('tendency_of_'): + continue + # auto-clone-constituents: framework-named std + # names (the whole constituent buffer / tendency + # buffer / properties array / count, and + # ``index_of_`` integers) resolve through the + # same ``source='constituent'`` channel but they + # are NOT individual constituent registrations — + # skip them so they don't get a spurious + # synthesised %instantiate call. + if (std in _FRAMEWORK_CONST_STDS + or std.startswith(_INDEX_PREFIX)): + continue + if std in seen: + continue + seen.add(std) + scheme_var = _lookup_scheme_var( + scheme_store, resolved_call.scheme_name, + resolved_call.phase, arg.scheme_local_name, + ) + if scheme_var is None: + # Should not happen — the resolver only + # produces ResolvedArg from a real MetaVar — + # but fail loudly rather than silently emit + # a malformed %instantiate. + raise CCPPError( + "auto-clone-constituents: cannot locate " + "scheme metadata for '{}' (scheme '{}', " + "phase '{}') while collecting auto-clone " + "entries".format( + arg.scheme_local_name, + resolved_call.scheme_name, + resolved_call.phase, + ) + ) + out.append(_make_auto_clone_entry(scheme_var)) + return out + +# auto-clone-constituents: END legacy-shim helpers. + + +def _collect_scheme_names(group) -> List[str]: + """Return ordered list of scheme names from a group, expanding subcycles/subcols.""" + names: List[str] = [] + from generator.suite_xml import SuiteScheme, SuiteSubcycle, SuiteSubcol + for item in group.items: + if isinstance(item, SuiteScheme): + names.append(item.name) + elif isinstance(item, (SuiteSubcycle, SuiteSubcol)): + for scheme_name in item.scheme_names(): + names.append(scheme_name) + return names + + +def _dedup_scheme_names(scheme_names: List[str]) -> List[str]: + """Return *scheme_names* with duplicates removed (first occurrence kept). + + Used by :func:`resolve_suite` for non-run phases: a scheme that appears + more than once in the suite XML still has its register/init/finalize + entry points invoked exactly once per group, while preserving the order + of first appearance. + """ + seen: Set[str] = set() + deduped: List[str] = [] + for scheme_name in scheme_names: + if scheme_name in seen: + continue + seen.add(scheme_name) + deduped.append(scheme_name) + return deduped + + +def _resolve_one_call( + scheme_name: str, + phase: str, + scheme_store, + host_dict: Dict[str, HostVarEntry], + suite_vars: Dict[str, 'SuiteVar'], + used_local_names: Set[str], + suite_name: str = '', + loop_context: Optional[List[Tuple[str, Optional[str]]]] = None, +) -> Optional[ResolvedCall]: + """Build a ResolvedCall for one scheme/phase, or return None if not defined. + + *loop_context* is a list of ``(loop_count_expr, loop_std_name)`` tuples + describing the enclosing ```` blocks, outermost first. Empty + when the call is not inside any subcycle. Forwarded to + :func:`_resolve_one_arg` so scheme args declaring ``ccpp_loop_counter`` + or ``ccpp_loop_extent`` can resolve against the generated loop locals. + """ + vars_list = scheme_store.variables_for(scheme_name, phase) + if vars_list is None: + return None + const_stds = scheme_store.constituent_stdnames() + resolved_call = ResolvedCall( + scheme_name=scheme_name, phase=phase, + scheme_module=scheme_store.module_for(scheme_name), + ) + for scheme_var in vars_list: + arg = _resolve_one_arg( + scheme_var, phase, host_dict, suite_vars, scheme_name, used_local_names, + suite_name=suite_name, loop_context=loop_context, + const_stds=const_stds, + ) + resolved_call.args.append(arg) + return resolved_call + + +def _resolve_flat_phase( + scheme_names: List[str], + phase: str, + scheme_store, + host_dict: Dict[str, HostVarEntry], + suite_vars: Dict[str, 'SuiteVar'], + used_local_names: Set[str], + suite_name: str = '', +) -> List[ResolvedCall]: + """Resolve a flat (non-subcycle) phase into a list of ResolvedCall.""" + result: List[ResolvedCall] = [] + for scheme_name in scheme_names: + resolved_call = _resolve_one_call(scheme_name, phase, scheme_store, host_dict, + suite_vars, used_local_names, + suite_name=suite_name) + if resolved_call is not None: + result.append(resolved_call) + return result + + +def _resolve_run_phase( + group, + phase: str, + scheme_store, + host_dict: Dict[str, HostVarEntry], + suite_vars: Dict[str, 'SuiteVar'], + used_local_names: Set[str], + suite_name: str = '', +) -> List[PhaseItem]: + """Resolve the run phase, preserving subcycle do-loop structure. + + :class:`SuiteScheme` items become :class:`ResolvedCall`. + :class:`SuiteSubcycle` items become :class:`ResolvedSubcycle`. When + a subcycle contains nested ```` elements, they are + preserved recursively so the cap emitter renders the corresponding + nested ``do`` loops (matches original capgen behaviour). + :class:`SuiteSubcol` items are flattened (treated as plain schemes). + """ + from generator.suite_xml import SuiteScheme, SuiteSubcycle, SuiteSubcol + + def _resolve_items( + suite_items, + loop_context: List[Tuple[str, Optional[str]]], + ) -> List[PhaseItem]: + """Recursively turn a list of SuiteScheme/SuiteSubcycle/SuiteSubcol + children into a list of :data:`PhaseItem`. Used at the top level + of a group AND for the body of every (possibly nested) subcycle. + + *loop_context* is the stack of enclosing ```` blocks + (outermost first), each as ``(loop_count_expr, loop_std_name)``. + Empty at the top level of a group; one entry per nested + subcycle depth. + """ + out: List[PhaseItem] = [] + for sub in suite_items: + if isinstance(sub, SuiteScheme): + resolved_call = _resolve_one_call( + sub.name, phase, scheme_store, host_dict, + suite_vars, used_local_names, + suite_name=suite_name, + loop_context=loop_context, + ) + if resolved_call is not None: + out.append(resolved_call) + elif isinstance(sub, SuiteSubcycle): + loop_count, loop_std = _resolve_subcycle_loop_bound( + sub.loop, host_dict, suite_vars=suite_vars, + ) + inner = _resolve_items( + sub.items, loop_context + [(loop_count, loop_std)], + ) + if inner: + out.append(ResolvedSubcycle( + loop=loop_count, calls=inner, + loop_std_name=loop_std, + )) + elif isinstance(sub, SuiteSubcol): + # SuiteSubcol is flattened in place — the framework + # doesn't render it as a separate loop level. + for scheme_name in sub.scheme_names(): + resolved_call = _resolve_one_call( + scheme_name, phase, scheme_store, host_dict, + suite_vars, used_local_names, + suite_name=suite_name, + loop_context=loop_context, + ) + if resolved_call is not None: + out.append(resolved_call) + return out + + result: List[PhaseItem] = [] + + for item in group.items: + if isinstance(item, SuiteScheme): + resolved_call = _resolve_one_call(item.name, phase, scheme_store, host_dict, + suite_vars, used_local_names, + suite_name=suite_name, + loop_context=[]) + if resolved_call is not None: + result.append(resolved_call) + elif isinstance(item, SuiteSubcycle): + loop_count, loop_std = _resolve_subcycle_loop_bound( + item.loop, host_dict, suite_vars=suite_vars, + ) + inner = _resolve_items( + item.items, [(loop_count, loop_std)], + ) + if inner: + result.append(ResolvedSubcycle( + loop=loop_count, calls=inner, + loop_std_name=loop_std, + )) + elif isinstance(item, SuiteSubcol): + for scheme_name in item.scheme_names(): + resolved_call = _resolve_one_call(scheme_name, phase, scheme_store, host_dict, + suite_vars, used_local_names, + suite_name=suite_name, + loop_context=[]) + if resolved_call is not None: + result.append(resolved_call) + + return result + + +def _collect_dim_uses( + resolved_group: ResolvedGroup, + host_dict: Dict[str, HostVarEntry], + suite_vars: Optional[Dict[str, 'SuiteVar']] = None, +) -> Dict[str, Set[str]]: + """Collect dimension variable USE requirements across all phases of a group. + + Host-module dimensions resolve to ``{host_module: {local_name}}``. + Suite-owned dimensions (set by ``_register``) resolve to + ``{ccpp__data: {ccpp_suite_data}}`` so the group cap can USE the + suite data module to access ``ccpp_suite_data(inst)%`` in dimension + expressions. + """ + dim_uses: Dict[str, Set[str]] = {} + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: + for dim_std in arg.used_dim_std_names: + entry = host_dict.get(dim_std) + if entry is not None and entry.module_name is not None: + mod = entry.module_name + # Walk back to the access-path root so DDT- + # component dims (e.g. ``physics%Model%levs``) + # USE the top-level instance (``physics``) and + # not the leaf (``levs``, which doesn't exist + # as a module symbol). Equivalent to + # ``entry.local_name`` for plain host vars. + sym = _root_symbol(entry.access_path) + dim_uses.setdefault(mod, set()).add(sym) + elif suite_vars and dim_std in suite_vars: + suite_var = suite_vars[dim_std] + dim_uses.setdefault(suite_var.module_name, set()).add( + 'ccpp_suite_data') + # Subcycle loop bounds resolved from CCPP standard names also need + # a USE entry (or, for control vars, a dummy arg — handled elsewhere). + # USE the *root* of the access path so DDT-component bounds pull + # in the parent instance (e.g. ``use mod, only: phys_state``) + # rather than the bare component name. Walk *every* subcycle in + # the phase, including nested ones — each level's bound must be + # in scope at do-loop emission time. + for item in iter_phase_subcycles(items): + if not item.loop_std_name: + continue + entry = host_dict.get(item.loop_std_name) + if entry is not None and entry.module_name is not None: + dim_uses.setdefault(entry.module_name, set()).add( + _root_symbol(entry.access_path) + ) + elif suite_vars and item.loop_std_name in suite_vars: + suite_var = suite_vars[item.loop_std_name] + dim_uses.setdefault(suite_var.module_name, set()).add( + 'ccpp_suite_data' + ) + return dim_uses diff --git a/capgen/generator/suite_types.py b/capgen/generator/suite_types.py new file mode 100644 index 00000000..73c5f6fd --- /dev/null +++ b/capgen/generator/suite_types.py @@ -0,0 +1,515 @@ +#!/usr/bin/env python3 + +"""Generate the shared pointer-wrapper types module for a suite. + +``ccpp__types.F90`` is a small Fortran module that declares one derived +type per unique (intrinsic-type, kind, rank) combination required by optional +arguments across all groups in the suite. Each generated type looks like:: + + type :: real_kind_phys_rank1_ptr_type + real(kind=kind_phys), pointer :: ptr(:) => null() + end type real_kind_phys_rank1_ptr_type + +Group cap modules USE this types module to declare optional-argument pointer +variables. + +Only generated when at least one optional argument is present. +""" + +import logging +import os +import re +from typing import Dict, List, Optional, Set, Tuple + +from metadata.parse_tools import CCPPError, open_if_changed +from generator.suite_resolver import SuiteResolution, iter_phase_calls + +_INDENT = ' ' + +# Fortran intrinsic types; anything else is treated as a DDT (or an +# ``external::`` reference, handled separately). +_INTRINSICS = frozenset({ + 'real', 'integer', 'character', 'logical', 'complex', 'double precision' +}) + + +def _is_intrinsic(type_: str) -> bool: + return type_.strip().lower() in _INTRINSICS + + +def _is_external(type_: str) -> bool: + return type_.strip().lower().startswith('external:') + + +def _split_external(type_: str) -> Tuple[str, str]: + """Parse ``external::`` into ``(module, typename)``.""" + parts = type_.strip().split(':', 2) + if len(parts) != 3: + raise CCPPError( + "Malformed external type spec '{}'; expected " + "'external::'".format(type_) + ) + return parts[1], parts[2] + + +######################################################################## +# Type-name helpers +######################################################################## + +def _ptr_type_name( + type_: str, + kind: str, + rank: int, + context: str = '', +) -> str: + """Return the Fortran derived-type name for a pointer wrapper. + + Parameters + ---------- + type_ : str + Fortran intrinsic type (e.g. ``'real'``, ``'integer'``) or DDT + type name (e.g. ``'cmpfsw_type'``). External types + (``'external::'``) are reduced to just + ```` for the wrapper name. + kind : str + Kind parameter (e.g. ``'kind_phys'``), or ``''`` if none. + rank : int + Number of array dimensions (0 = scalar). + context : str, optional + Free-form prefix passed through to :func:`_sanitize_len_suffix` + and prepended to any error message raised from there. Callers + identify the offending scheme + argument here so the user can + locate the metadata block to fix; see + :func:`_ptr_type_name_for_arg`. + + Returns + ------- + str + + Examples + -------- + >>> _ptr_type_name('integer', '', 1) + 'integer_rank1_ptr_type' + >>> _ptr_type_name('real', 'kind_phys', 1) + 'real_kind_phys_rank1_ptr_type' + >>> _ptr_type_name('real', '', 0) + 'real_rank0_ptr_type' + >>> _ptr_type_name('real', 'kind_phys', 2) + 'real_kind_phys_rank2_ptr_type' + >>> _ptr_type_name('cmpfsw_type', '', 1) + 'cmpfsw_type_rank1_ptr_type' + >>> _ptr_type_name('external:mpi_f08:mpi_comm', '', 0) + 'mpi_comm_rank0_ptr_type' + >>> _ptr_type_name('character', 'len=10', 1) + 'character_len10_rank1_ptr_type' + >>> _ptr_type_name('character', 'len=3', 1) + 'character_len3_rank1_ptr_type' + """ + if _is_external(type_): + _, typename = _split_external(type_) + name = typename + else: + name = type_ + parts = [name] + if kind: + if kind.startswith('len='): + # Different character lengths require *different* wrapper + # types — a DDT component can't be ``character(len=*)``, so + # the wrapper name must encode the length spec. Sanitise + # the suffix because raw lengths like ``:`` (deferred) or + # ``*`` (assumed) aren't valid Fortran identifier chars, + # and would otherwise produce illegal type names like + # ``character_len:_rank1_ptr_type`` that the compiler + # rejects. + len_spec = kind[len('len='):].strip() + parts.append('len' + _sanitize_len_suffix(len_spec, context=context)) + else: + parts.append(kind) + parts.append('rank{}'.format(rank)) + parts.append('ptr_type') + return '_'.join(parts) + + +def _sanitize_len_suffix(len_spec: str, context: str = '') -> str: + """Return a Fortran-identifier-safe suffix for a ``character(len=…)`` spec. + + Pointer-wrapper type names embed the length specifier, e.g. + ``character_len10_rank1_ptr_type``. Raw length specs include + forms that aren't valid Fortran identifier characters: + + * ``len=N`` (positive integer literal) — already safe. + * ``len=:`` (deferred length, paired with ``pointer`` / ``allocatable``) — + replaced with ``_deferred``. + * ``len=*`` (assumed length) — REJECTED: assumed-length is not + legal as a DDT component spec. Raises :class:`CCPPError`. + * ``len=NAME`` (Fortran parameter or constant symbol) — kept verbatim + when it is already a valid identifier. + + Anything that doesn't fit those forms raises ``CCPPError`` rather + than silently producing an illegal Fortran identifier. + + *context* — when non-empty, prefixed to every error message as + ``": ..."``. Callers identify the offending scheme + + argument so the user can locate the metadata block to fix. + """ + prefix = '{}: '.format(context) if context else '' + spec = len_spec.strip() + if not spec: + raise CCPPError( + "{}Empty character length specifier 'len=' in pointer-wrapper " + "type name construction; expected an integer literal, " + "a parameter name, or ':' for deferred length.".format(prefix) + ) + if spec == ':': + return '_deferred' + if spec == '*': + raise CCPPError( + "{}character(len=*) cannot appear as a DDT component, so " + "capgen cannot generate a pointer-wrapper type for it. " + "Use a concrete length, a parameter constant, or 'len=:' " + "(deferred length, paired with allocatable / pointer) " + "in the metadata instead.".format(prefix) + ) + # Plain integer literal (digits) or Fortran identifier — accept verbatim. + if spec.isdigit() or _IDENT_RE.match(spec): + return spec + raise CCPPError( + "{}Cannot derive a Fortran-identifier-safe pointer-wrapper type " + "name from 'len={}'. Expected an integer literal, a parameter " + "identifier, or ':' (deferred length).".format(prefix, spec) + ) + + +_IDENT_RE = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$') + + +def _ptr_rank(arg) -> int: + """Return the effective rank of the value pointed to by *arg*'s pointer. + + For optional args without transform (Case 2) or with transform (Case 4), + the pointer rank equals the number of dimensions of the host/suite variable. + """ + if arg.host_entry is not None: + return len(arg.host_entry.dimensions) + if arg.suite_var is not None: + return len(arg.suite_var.dimensions) + return 0 + + +def _ptr_type_for_arg(arg) -> Tuple[str, str, int]: + """Return the (type_, kind, rank) tuple for *arg*'s pointer wrapper. + + For Case 2 (optional, no transform), the pointer targets the host + variable directly — same type and kind. For Case 4 (optional + + transform), the pointer targets the transformation temporary, which + carries the scheme's kind. + + Narrow override for ``character(len=*)``: the resolver explicitly + accepts scheme ``kind=len=*`` paired with host ``kind=len=N`` (or + ``len=:``) as compatible and emits no kind transform — so we are + always in Case 2 here, and the pointer points at host data with a + concrete length. The DDT-component rules forbid ``len=*``, but the + host's concrete kind is usable verbatim, so fall through to the + host kind in that one situation. All other types keep the + scheme-wins precedence — Case 4 transform temps need the scheme + kind, and ``character`` is the only intrinsic where ``len=*`` + survives the resolver as an unconverted compatibility. + """ + if arg.host_entry is not None: + type_ = arg.host_entry.type + else: + type_ = arg.suite_var.type_ + host_kind = (arg.host_entry.kind if arg.host_entry + else arg.suite_var.kind) + if type_ == 'character' and arg.kind_scheme == 'len=*': + kind = host_kind + else: + kind = arg.kind_scheme or host_kind + rank = _ptr_rank(arg) + return type_, kind, rank + + +def _ptr_type_name_for_arg(arg, scheme_name: str) -> str: + """Build the pointer-wrapper type name for *arg* with rich error context. + + Resolves ``(type_, kind, rank)`` via :func:`_ptr_type_for_arg`, then + delegates to :func:`_ptr_type_name` passing a *context* string of + the form ``"scheme '', optional argument [] + (standard_name=, intent=)"``. Any :class:`CCPPError` from + the name-construction path then carries enough information for the + user to find the offending metadata block without grepping. + + Use this at every site that calls :func:`_ptr_type_name` on a + resolved scheme argument. Bare :func:`_ptr_type_name` calls are + only appropriate when no per-argument context is available (e.g. + iterating the already-validated combo set in + :func:`_generate_suite_types`). + """ + type_, kind, rank = _ptr_type_for_arg(arg) + context = ( + "scheme '{}', optional argument [{}] " + "(standard_name={}, intent={})".format( + scheme_name, arg.scheme_local_name, + arg.standard_name, arg.intent, + ) + ) + return _ptr_type_name(type_, kind, rank, context=context) + + +######################################################################## +# Collection helpers +######################################################################## + +def _collect_ptr_type_combos( + suite_res: SuiteResolution, +) -> Set[Tuple[str, str, int]]: + """Collect unique (type, kind, rank) tuples needed by optional args. + + Validates each combo eagerly via :func:`_ptr_type_name_for_arg` so + that any unsupported shape (notably ``character(len=*)``) raises a + :class:`CCPPError` carrying the offending scheme + argument name + here, rather than later from the bare-tuple loop in + :func:`_generate_suite_types` (which would have nothing useful to + say to the user). The constructed name itself is discarded — only + the validation matters at this point. + + Parameters + ---------- + suite_res : SuiteResolution + + Returns + ------- + set of (type_, kind, rank) + """ + combos: Set[Tuple[str, str, int]] = set() + for resolved_group in suite_res.groups: + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: + if arg.ptr_name: + _ptr_type_name_for_arg(arg, resolved_call.scheme_name) + combos.add(_ptr_type_for_arg(arg)) + return combos + + +######################################################################## +# Module generator +######################################################################## + +def _fortran_type_str_simple(type_: str, kind: str) -> str: + """Build a Fortran type-clause for a pointer-wrapper declaration. + + Handles three categories: + + * **Intrinsics** (``real``, ``integer``, ``character``, ...) — emit + with optional ``(kind=...)`` (or ``(len=...)`` for character). + * **DDT types** (anything not an intrinsic and not ``external:`` — + e.g. ``cmpfsw_type``) — wrap in ``type(...)`` so the declaration + is a syntactically valid Fortran derived-type reference. Kind is + not meaningful for DDTs. + * **External types** (``external::``) — emit + ``type()``. The defining module is USE'd separately + (see :func:`_collect_ddt_uses`). + + >>> _fortran_type_str_simple('real', 'kind_phys') + 'real(kind=kind_phys)' + >>> _fortran_type_str_simple('integer', '') + 'integer' + >>> _fortran_type_str_simple('character', 'len=512') + 'character(len=512)' + >>> _fortran_type_str_simple('cmpfsw_type', '') + 'type(cmpfsw_type)' + >>> _fortran_type_str_simple('external:mpi_f08:mpi_comm', '') + 'type(mpi_comm)' + """ + t = type_.strip() + if _is_external(t): + _, typename = _split_external(t) + return 'type({})'.format(typename) + if not _is_intrinsic(t): + # DDT — Fortran requires the type(...) wrapper. Kind is not + # meaningful here; drop it. + return 'type({})'.format(t) + if kind: + if t.lower().startswith('character'): + return 'character({})'.format(kind) + return '{}(kind={})'.format(t, kind) + return t + + +def _collect_ddt_uses( + combos: Set[Tuple[str, str, int]], + ddt_module_map: Optional[Dict[str, str]], +) -> Dict[str, Set[str]]: + """Group DDT and external types referenced in *combos* by USE module. + + Intrinsics are skipped (they need no USE). DDT types are looked up + in *ddt_module_map* (built by + :func:`metadata.variable_resolver.build_ddt_module_map`); external + types parse their module out of the ``external::`` + prefix. + + Returns + ------- + dict mapping ``module_name -> {typename, ...}``. + + Raises + ------ + CCPPError + If a DDT referenced by a pointer wrapper is absent from + *ddt_module_map* — the generator can't emit a valid USE for it. + """ + uses: Dict[str, Set[str]] = {} + for type_, _kind, _rank in combos: + t = type_.strip() + if _is_intrinsic(t): + continue + if _is_external(t): + mod, typename = _split_external(t) + uses.setdefault(mod, set()).add(typename) + continue + # DDT — look up the defining Fortran module. + if ddt_module_map is None or t not in ddt_module_map: + raise CCPPError( + "Pointer wrapper needs DDT '{}' but its defining Fortran " + "module is unknown. Declare it explicitly with " + "'module_name = ' in the '{}' DDT's " + "[ccpp-table-properties], or co-locate the DDT table with a " + "scheme/host/control table in the same .meta file.".format( + t, t, + ) + ) + uses.setdefault(ddt_module_map[t], set()).add(t) + return uses + + +def _dim_spec(rank: int) -> str: + """Return the deferred-shape dimension specifier for a *rank*-dimensional pointer. + + >>> _dim_spec(0) + '' + >>> _dim_spec(1) + '(:)' + >>> _dim_spec(2) + '(:,:)' + """ + if rank == 0: + return '' + return '({})'.format(','.join([':'] * rank)) + + +def _generate_suite_types( + suite_name: str, + combos: Set[Tuple[str, str, int]], + ddt_module_map: Optional[Dict[str, str]] = None, +) -> List[str]: + """Generate the Fortran source lines for the suite types module. + + Parameters + ---------- + suite_name : str + combos : set of (type_, kind, rank) + ddt_module_map : dict, optional + DDT type name → defining Fortran module. Required when any + ``combos`` entry's *type_* is a DDT — the module is USE'd so + the ``type()`` reference resolves. + + Returns + ------- + list of str (without trailing newlines) + """ + mod_name = 'ccpp_{}_types'.format(suite_name) + lines: List[str] = [] + + lines.append( + '! {}.F90 -- generated by ccpp_capgen, do not edit'.format(mod_name) + ) + lines.append('module {}'.format(mod_name)) + lines.append('') + + # USE ccpp_kinds for any kind parameters referenced in pointer-wrapper + # type declarations (e.g. ``real(kind=kind_phys)``). + kind_names = sorted({ + kind for _t, kind, _r in combos + if kind and not kind.startswith('len=') + }) + if kind_names: + lines.append( + '{}use ccpp_kinds, only: {}'.format(_INDENT, ', '.join(kind_names)) + ) + + # USE the defining module for every DDT (or external) type the + # pointer wrappers reference, so ``type()`` resolves. + ddt_uses = _collect_ddt_uses(combos, ddt_module_map) + for mod in sorted(ddt_uses): + symbols = ', '.join(sorted(ddt_uses[mod])) + lines.append('{}use {}, only: {}'.format(_INDENT, mod, symbols)) + if kind_names or ddt_uses: + lines.append('') + + lines.append('{}implicit none'.format(_INDENT)) + lines.append('{}private'.format(_INDENT)) + lines.append('') + + sorted_combos = sorted(combos) + + for type_, kind, rank in sorted_combos: + tname = _ptr_type_name(type_, kind, rank) + lines.append('{}public :: {}'.format(_INDENT, tname)) + + lines.append('') + + for type_, kind, rank in sorted_combos: + tname = _ptr_type_name(type_, kind, rank) + ftype = _fortran_type_str_simple(type_, kind) + dimspec = _dim_spec(rank) + lines.append('{}type :: {}'.format(_INDENT, tname)) + lines.append( + '{} {}, pointer :: ptr{} => null()'.format(_INDENT, ftype, dimspec) + ) + lines.append('{}end type {}'.format(_INDENT, tname)) + lines.append('') + + lines.append('end module {}'.format(mod_name)) + return lines + + +######################################################################## +# Public API +######################################################################## + +def write_suite_types( + suite_name: str, + suite_res: SuiteResolution, + output_root: str, + ddt_module_map: Optional[Dict[str, str]] = None, + logger: Optional[logging.Logger] = None, +) -> Optional[str]: + """Write the suite types module to *output_root*. + + Does nothing and returns ``None`` when the suite has no optional arguments. + + Parameters + ---------- + suite_name : str + suite_res : SuiteResolution + output_root : str + Output directory (created if absent). + + Returns + ------- + str or None + Absolute path of the written file, or ``None`` if nothing was written. + """ + combos = _collect_ptr_type_combos(suite_res) + if not combos: + return None + + os.makedirs(output_root, exist_ok=True) + filename = 'ccpp_{}_types.F90'.format(suite_name) + out_path = os.path.join(output_root, filename) + + lines = _generate_suite_types(suite_name, combos, ddt_module_map) + with open_if_changed(out_path, logger=logger) as fh: + fh.write('\n'.join(lines) + '\n') + return out_path diff --git a/capgen/generator/suite_xml.py b/capgen/generator/suite_xml.py new file mode 100644 index 00000000..fb8f50e7 --- /dev/null +++ b/capgen/generator/suite_xml.py @@ -0,0 +1,646 @@ +#!/usr/bin/env python3 + +"""Suite Definition File (SDF) parser for ccpp-capgen. + +A Suite Definition File is an XML document that describes which physics +schemes to run, in which order, and how to group them. This module: + +1. Reads and schema-validates the SDF (v1.0 or v2.0). +2. For v2.0 SDFs: expands ```` references recursively. +3. Writes the expanded XML to ``/ccpp__expanded.xml``. +4. Builds an in-memory :class:`Suite` object that the code generator uses. + +Suite XML format (v2.0) +----------------------- +:: + + + + + + suite_init_scheme + + + scheme_a + + + + + scheme_b + + + + + + + + + suite_final_scheme + + + +Backward compatibility +---------------------- +* Schema v1.0 suites are accepted (no nested-suite expansion). +* The old element spellings ```` (typo) and ```` are + accepted but a :mod:`logging` warning is emitted directing the author to + use ```` / ```` instead. + +Expanded XML file +----------------- +After expanding all ```` references the result is written as:: + + /ccpp__expanded.xml + +This file is for human inspection only; it is not consumed by subsequent +generator runs. + +Schema location +--------------- +The XSD files live in ``capgen/schema/``. Their path is resolved relative +to this source file at runtime, so no extra configuration is needed. +""" + +import logging +import os +import xml.etree.ElementTree as ET +from typing import Dict, List, Optional, Union + +from metadata.parse_tools import ( + CCPPError, + read_xml_file, + find_schema_version, + expand_nested_suites, + write_xml_file, +) +from metadata.parse_tools.xml_tools import validate_xml_file + +######################################################################## +# Constants +######################################################################## + +#: Directory containing the XSD schemas, relative to this source file. +_SCHEMA_DIR = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + 'schema', +) + +#: Accepted ```` element name. The old ccpp-prebuild schema also +#: tolerated ```` (typo) and ````; capgen +#: rejects both with a hard error so SDFs migrate to the canonical +#: short name. +_INIT_TAG = 'init' + +#: Accepted ```` element name. The old ccpp-prebuild schema +#: also tolerated ````; capgen rejects it. +_FINAL_TAG = 'final' + +#: Element names that were valid in the old schema but are rejected in +#: capgen's v2.0 SDF format. Map to the canonical short form so the +#: error message can point at the right replacement. +_REJECTED_INIT_TAGS = frozenset({'initalize', 'initialize'}) +_REJECTED_FINAL_TAGS = frozenset({'finalize'}) + + +######################################################################## +# In-memory suite data model +######################################################################## + +class SuiteScheme: + """A single scheme call within a group or subcycle. + + Parameters + ---------- + name : str + Scheme name — must match a ``type = scheme`` metadata table. + + Examples + -------- + >>> SuiteScheme('my_scheme').name + 'my_scheme' + """ + + def __init__(self, name: str): + self.name: str = name.strip() + + def scheme_names(self) -> List[str]: + """Return this scheme's name in a one-element list.""" + return [self.name] + + def __repr__(self) -> str: + return f"SuiteScheme({self.name!r})" + + +class SuiteSubcycle: + """A ```` loop wrapping one or more scheme call sites. + + Parameters + ---------- + loop : str or None + Loop-count value as written in the XML ``loop`` attribute. + May be: + + * An integer literal (e.g. ``"2"`` or ``"10"``) — a compile-time + constant. + * A Fortran identifier (e.g. ``"num_subcycles_for_scheme6"``) — a + CCPP standard name resolved against control variable metadata at + cap-generation time; becomes the value of ``ccpp_loop_extent`` + inside the loop. + * ``None`` if the attribute is absent (treated as a single iteration). + + items : list + Ordered sequence of :class:`SuiteScheme`, :class:`SuiteSubcycle`, + or :class:`SuiteSubcol` children. + + Examples + -------- + >>> subcycle = SuiteSubcycle(loop='2', items=[SuiteScheme('sch_a')]) + >>> subcycle.loop + '2' + >>> subcycle.is_literal_count + True + >>> subcycle2 = SuiteSubcycle(loop='num_subcycles_for_ag', items=[]) + >>> subcycle2.is_literal_count + False + """ + + def __init__(self, loop: Optional[str], items: list): + self.loop: Optional[str] = loop.strip() if loop else None + self.items: list = items + + @property + def is_literal_count(self) -> bool: + """Return True if the loop count is an integer literal.""" + if self.loop is None: + return True + try: + int(self.loop) + return True + except ValueError: + return False + + def scheme_names(self) -> List[str]: + """Return all scheme names referenced inside this subcycle.""" + names = [] + for item in self.items: + names.extend(item.scheme_names()) + return names + + def __repr__(self) -> str: + return f"SuiteSubcycle(loop={self.loop!r}, nitems={len(self.items)})" + + +class SuiteSubcol: + """A ```` sub-column processing element. + + Sub-column processing uses a ``gen`` routine to generate sub-columns + from the GCM column and an ``avg`` routine to average them back. + + Parameters + ---------- + gen_routine : str + Fortran identifier of the sub-column generation routine. + avg_routine : str + Fortran identifier of the sub-column averaging routine. + items : list + Ordered sequence of children (schemes and/or subcycles). + """ + + def __init__(self, gen_routine: str, avg_routine: str, items: list): + self.gen_routine: str = gen_routine.strip() + self.avg_routine: str = avg_routine.strip() + self.items: list = items + + def scheme_names(self) -> List[str]: + """Return all scheme names referenced inside this subcol.""" + names = [] + for item in self.items: + names.extend(item.scheme_names()) + return names + + def __repr__(self) -> str: + return (f"SuiteSubcol(gen={self.gen_routine!r}, " + f"avg={self.avg_routine!r}, nitems={len(self.items)})") + + +# Type alias for any element that can appear inside a group. +GroupItem = Union[SuiteScheme, SuiteSubcycle, SuiteSubcol] + + +class SuiteGroup: + """A named ```` within a suite. + + Parameters + ---------- + name : str + Group name (must be a valid Fortran identifier, unique within suite). + items : list of GroupItem + Ordered sequence of scheme calls, subcycles, and subcol elements. + + Examples + -------- + >>> grp = SuiteGroup('dynamics', [SuiteScheme('dyn_scheme')]) + >>> grp.name + 'dynamics' + >>> grp.scheme_names() + ['dyn_scheme'] + """ + + def __init__(self, name: str, items: List[GroupItem]): + self.name: str = name.strip() + self.items: List[GroupItem] = items + + def scheme_names(self) -> List[str]: + """Return all unique scheme names called in this group (ordered, no dedup).""" + names = [] + for item in self.items: + names.extend(item.scheme_names()) + return names + + def unique_scheme_names(self) -> List[str]: + """Return unique scheme names preserving first-occurrence order.""" + seen = set() + result = [] + for name in self.scheme_names(): + if name not in seen: + seen.add(name) + result.append(name) + return result + + def __repr__(self) -> str: + return f"SuiteGroup({self.name!r}, nitems={len(self.items)})" + + +class Suite: + """In-memory representation of a fully-parsed and expanded SDF. + + Parameters + ---------- + name : str + Suite name from the ``name`` attribute of ````. + version : list of int + Schema version ``[major, minor]``. + source_file : str + Absolute path to the original ``.xml`` source file. + groups : list of SuiteGroup + Ordered list of groups. + init_scheme : str or None + Name of the suite-level init scheme (```` element), or ``None``. + final_scheme : str or None + Name of the suite-level final scheme (```` element), or ``None``. + expanded_file : str or None + Path to the written expanded XML file, set after :func:`parse_suite_xml` + writes it. + + Examples + -------- + >>> g = SuiteGroup('grp', [SuiteScheme('sch')]) + >>> s = Suite('my_suite', [2, 0], '/path/to/f.xml', [g], None, None) + >>> s.name + 'my_suite' + >>> s.group_names() + ['grp'] + >>> s.all_scheme_names() + ['sch'] + """ + + def __init__( + self, + name: str, + version: List[int], + source_file: str, + groups: List[SuiteGroup], + init_scheme: Optional[str], + final_scheme: Optional[str], + expanded_file: Optional[str] = None, + ): + self.name: str = name + self.version: List[int] = version + self.source_file: str = source_file + self.groups: List[SuiteGroup] = groups + self.init_scheme: Optional[str] = init_scheme + self.final_scheme: Optional[str] = final_scheme + self.expanded_file: Optional[str] = expanded_file + + def group_names(self) -> List[str]: + """Return the group names in declaration order.""" + return [g.name for g in self.groups] + + def get_group(self, name: str) -> Optional[SuiteGroup]: + """Return the group with *name*, or ``None``.""" + for grp in self.groups: + if grp.name == name: + return grp + return None + + def all_scheme_names(self) -> List[str]: + """Return all unique scheme names across all groups (first-occurrence order).""" + seen = set() + result = [] + for grp in self.groups: + for name in grp.scheme_names(): + if name not in seen: + seen.add(name) + result.append(name) + return result + + def __repr__(self) -> str: + return (f"Suite({self.name!r}, version={self.version}, " + f"ngroups={len(self.groups)})") + + +######################################################################## +# XML-to-object conversion +######################################################################## + +def _parse_group_items(parent: ET.Element) -> List[GroupItem]: + """Parse the child elements of a ```` or ````/```` + into a list of :class:`GroupItem` objects. + + Parameters + ---------- + parent : xml.etree.ElementTree.Element + The containing XML element. + + Returns + ------- + list of GroupItem + """ + items: List[GroupItem] = [] + for child in parent: + tag = child.tag.lower() + if tag == 'scheme': + name = (child.text or '').strip() + if not name: + raise CCPPError( + f"Empty element inside <{parent.tag}>" + ) + items.append(SuiteScheme(name)) + elif tag == 'subcycle': + loop_val = child.get('loop') + sub_items = _parse_group_items(child) + items.append(SuiteSubcycle(loop=loop_val, items=sub_items)) + elif tag == 'subcol': + gen = child.get('gen', '').strip() + avg = child.get('avg', '').strip() + if not gen or not avg: + raise CCPPError( + " requires both 'gen' and 'avg' attributes" + ) + sub_items = _parse_group_items(child) + items.append(SuiteSubcol(gen, avg, sub_items)) + # Anything else (whitespace text nodes, comments) is silently ignored + # after nested_suite expansion (those elements are already gone). + return items + + +def _build_suite(root: ET.Element, source_file: str, + version: List[int], logger: logging.Logger) -> Suite: + """Build a :class:`Suite` from an *expanded* XML root element. + + This function must be called after :func:`expand_nested_suites` has + already resolved all ```` references. + + Parameters + ---------- + root : xml.etree.ElementTree.Element + The ```` root element (expanded). + source_file : str + Path to the original ``.xml`` file (for error messages). + version : list of int + Schema version ``[major, minor]``. + logger : logging.Logger + + Returns + ------- + Suite + """ + suite_name = root.get('name', '').strip() + if not suite_name: + raise CCPPError( + f"Suite XML '{source_file}' is missing the 'name' attribute " + "on the element" + ) + + init_scheme: Optional[str] = None + final_scheme: Optional[str] = None + groups: List[SuiteGroup] = [] + + for child in root: + tag = child.tag.lower() + + if tag == _INIT_TAG: + name = (child.text or '').strip() + if not name: + raise CCPPError( + f"SDF '{source_file}': empty <{child.tag}> element" + ) + init_scheme = name + + elif tag in _REJECTED_INIT_TAGS: + raise CCPPError( + f"SDF '{source_file}': element <{child.tag}> is not " + f"accepted; use the short form <{_INIT_TAG}> " + f"(single scheme name as text content)." + ) + + elif tag == _FINAL_TAG: + name = (child.text or '').strip() + if not name: + raise CCPPError( + f"SDF '{source_file}': empty <{child.tag}> element" + ) + final_scheme = name + + elif tag in _REJECTED_FINAL_TAGS: + raise CCPPError( + f"SDF '{source_file}': element <{child.tag}> is not " + f"accepted; use the short form <{_FINAL_TAG}> " + f"(single scheme name as text content)." + ) + + elif tag == 'group': + grp_name = child.get('name', '').strip() + if not grp_name: + raise CCPPError( + f"SDF '{source_file}': is missing 'name' attribute" + ) + items = _parse_group_items(child) + groups.append(SuiteGroup(grp_name, items)) + + elif tag == 'nested_suite': + # Should not occur after expansion; warn and skip. + logger.warning( + "SDF '%s': unexpanded element found after " + "expansion — it will be ignored.", source_file + ) + + # else: whitespace / unknown elements — silently ignored + + if not groups: + logger.warning( + "SDF '%s': suite '%s' contains no elements.", + source_file, suite_name + ) + + # Check for duplicate group names (schema enforces xs:ID uniqueness but + # validation may be skipped or xmllint may not be installed). + seen_groups: Dict[str, bool] = {} + for grp in groups: + if grp.name in seen_groups: + raise CCPPError( + f"SDF '{source_file}': duplicate group name '{grp.name}' " + f"in suite '{suite_name}'" + ) + seen_groups[grp.name] = True + + return Suite( + name=suite_name, + version=version, + source_file=source_file, + groups=groups, + init_scheme=init_scheme, + final_scheme=final_scheme, + ) + + +######################################################################## +# Public API +######################################################################## + +def parse_suite_xml( + suite_file: str, + output_root: str, + logger: Optional[logging.Logger] = None, + schema_path: Optional[str] = None, + skip_validation: bool = False, +) -> Suite: + """Parse a Suite Definition File, expand nested suites, and return a + :class:`Suite` object. + + Processing steps: + + 1. Read and XML-parse the file. + 2. Extract the schema version. + 3. Validate against the bundled XSD (unless *skip_validation* is set). + 4. For v2 suites: expand all ```` references. + 5. Re-validate the expanded XML. + 6. Write the expanded XML to + ``/ccpp__expanded.xml``. + 7. Build and return the in-memory :class:`Suite` object. + + Parameters + ---------- + suite_file : str + Path to the ``.xml`` SDF. + output_root : str + Directory where the expanded XML is written. Created if absent. + logger : logging.Logger, optional + Logger. A module-level logger is used if ``None``. + schema_path : str, optional + Directory containing XSD files. Defaults to the bundled + ``capgen/schema/`` directory. + skip_validation : bool + If ``True``, skip XML schema validation (useful in test environments + where ``xmllint`` is not available). + + Returns + ------- + Suite + Fully parsed suite with all nested suites expanded. + + Raises + ------ + CCPPError + On any structural, schema, or content error. + + Examples + -------- + Parse a simple suite without writing to disk (using *skip_validation* + and a temp directory):: + + suite = parse_suite_xml('my_suite.xml', '/tmp/capgen_out', + skip_validation=True) + print(suite.name) + print(suite.group_names()) + """ + log = logger or logging.getLogger(__name__) + sdir = schema_path or _SCHEMA_DIR + + if not os.path.isfile(suite_file): + raise CCPPError(f"Suite XML file '{suite_file}' does not exist") + + log.info("Reading suite XML: %s", suite_file) + _, root = read_xml_file(suite_file, log) + try: + version = find_schema_version(root) + except CCPPError as verr: + raise CCPPError( + f"{verr} in suite XML file '{suite_file}'" + ) from verr + log.debug("Suite XML schema version: %d.%d", *version) + + # ---- schema validation (pre-expansion) -------------------------------- + if not skip_validation: + validate_xml_file(suite_file, 'suite', version, log, schema_path=sdir) + + # ---- expand nested suites (v2 only) ----------------------------------- + if version[0] >= 2: + suite_dir = os.path.dirname(os.path.abspath(suite_file)) + expand_nested_suites(root, suite_dir, logger=log) + + # ---- build in-memory Suite object ------------------------------------- + suite = _build_suite(root, suite_file, version, log) + + # ---- write expanded XML to output_root -------------------------------- + os.makedirs(output_root, exist_ok=True) + expanded_name = f"ccpp_{suite.name}_expanded.xml" + expanded_path = os.path.join(output_root, expanded_name) + # write_xml_file logs "Wrote " or "Unchanged: " via + # write_if_changed when the logger is provided — no need to duplicate + # here. + write_xml_file(root, expanded_path, log) + suite.expanded_file = expanded_path + + # ---- re-validate the expanded XML (catches duplicate xs:ID errors) ---- + if not skip_validation: + validate_xml_file(expanded_path, 'suite', version, log, schema_path=sdir) + + return suite + + +def parse_suite_xml_files( + suite_files: List[str], + output_root: str, + logger: Optional[logging.Logger] = None, + schema_path: Optional[str] = None, + skip_validation: bool = False, +) -> List[Suite]: + """Parse a list of SDF files and return a :class:`Suite` per file. + + Wrapper around :func:`parse_suite_xml` for processing multiple suites + in one call. + + Parameters + ---------- + suite_files : list of str + Paths to ``.xml`` SDF files. + output_root : str + Passed to :func:`parse_suite_xml`. + logger : logging.Logger, optional + schema_path : str, optional + skip_validation : bool + + Returns + ------- + list of Suite + """ + suites = [] + for fpath in suite_files: + suites.append(parse_suite_xml( + fpath, output_root, + logger=logger, schema_path=schema_path, + skip_validation=skip_validation, + )) + return suites diff --git a/capgen/generator/trace.py b/capgen/generator/trace.py new file mode 100644 index 00000000..7a7c146d --- /dev/null +++ b/capgen/generator/trace.py @@ -0,0 +1,205 @@ +"""Trace-emission helpers shared by all cap generators. + +A generated cap module carries a compile-time gate:: + + use, intrinsic :: iso_fortran_env, only: error_unit + logical, parameter :: trace = .false. + +and every cap subroutine that has at least one ``intent(in)``/ +``intent(inout)`` control dummy emits a guarded write as the very first +line of its body:: + + if (trace) write(error_unit, '(a,a,a,a,1x,i0)') & + 'CCPP TRACE :', & + ' =', trim(), & + ' =', , & + ... + +The format string is built per-call: each character item contributes an +``a`` descriptor (the label literal and any ``trim()``-wrapped value), +and each integer item contributes ``1x,i0`` so the value is printed +flush against a single space separator rather than the wide default +field of list-directed I/O. + +Two effects: + +1. With ``trace = .false.`` (the default) the compiler eliminates the + write at the dead branch, but every control dummy is *syntactically* + referenced inside the dead block. This silences "unused dummy + argument" warnings on strict compilers (Intel oneAPI in particular) + without any runtime cost. +2. Flipping the parameter to ``.true.`` -- either via the ``--trace`` CLI + flag at generation time, or by hand-editing one generated file -- + turns the cap into a self-describing trace, useful for diagnosing + call-chain problems in a host integration. + +The helpers below are pure (no I/O, no side effects on the caller) and +return lists of Fortran source lines with no trailing newlines. +""" + +from typing import Iterable, List, Optional + + +def emit_module_gate(trace_default: bool, indent: str) -> List[str]: + """Return the module-scope ``trace`` parameter declaration lines. + + The caller is responsible for ensuring ``use, intrinsic :: iso_fortran_env, + only: error_unit`` is present in the module USE list (see + :func:`ensure_error_unit_use`). This helper only emits the + ``logical, parameter`` line so it can be placed alongside other + module-scope parameters. + + Parameters + ---------- + trace_default : bool + ``True`` -> emit ``logical, parameter :: trace = .true.``; + ``False`` -> ``.false.``. + indent : str + Leading whitespace for each emitted line. + + Returns + ------- + list of str + One line, no trailing newline. + + Examples + -------- + >>> emit_module_gate(False, ' ') + [' logical, parameter :: trace = .false.'] + >>> emit_module_gate(True, ' ') + [' logical, parameter :: trace = .true.'] + """ + value = '.true.' if trace_default else '.false.' + return ['{}logical, parameter :: trace = {}'.format(indent, value)] + + +def ensure_error_unit_use(use_lines: List[str], indent: str) -> List[str]: + """Insert an ``iso_fortran_env`` USE for ``error_unit`` if not present. + + The trace block writes to ``error_unit``, which must be visible inside + each cap module. Callers that already build a USE list pass it in; + this helper appends the required line iff no existing line references + ``error_unit`` (matched as a whole word). + + Parameters + ---------- + use_lines : list of str + Existing module-level USE lines (modified in place and returned). + indent : str + Leading whitespace for the emitted line. + + Returns + ------- + list of str + The same list, possibly with one extra line appended. + + Examples + -------- + >>> ensure_error_unit_use([], ' ') + [' use, intrinsic :: iso_fortran_env, only: error_unit'] + >>> ensure_error_unit_use([' use foo, only: bar'], ' ') + [' use foo, only: bar', ' use, intrinsic :: iso_fortran_env, only: error_unit'] + + Idempotent when ``error_unit`` is already there: + + >>> ensure_error_unit_use( + ... [' use, intrinsic :: iso_fortran_env, only: error_unit'], ' ' + ... ) + [' use, intrinsic :: iso_fortran_env, only: error_unit'] + """ + for line in use_lines: + # Whole-word match so we don't false-positive on a substring. + tokens = line.replace(',', ' ').replace(':', ' ').split() + if 'error_unit' in tokens: + return use_lines + use_lines.append( + '{}use, intrinsic :: iso_fortran_env, only: error_unit'.format(indent) + ) + return use_lines + + +def emit_trace_block( + sub_name: str, + ctrl_entries: Iterable, + indent: str, + instance_local: Optional[str] = None, + extra_in_names: Optional[Iterable[str]] = None, +) -> List[str]: + """Return the gated trace ``write`` lines for one cap subroutine. + + The trace lists every control dummy whose intent is ``in`` or + ``inout`` (i.e. the dummies that a strict compiler would otherwise + flag as unused). ``intent(out)`` dummies (``ccpp_error_code`` / + ``ccpp_error_message``) are excluded so the write can sit at the + very first body line, before any initialisation, with no risk of + reading uninitialised storage. + + Parameters + ---------- + sub_name : str + Fully-qualified Fortran name of the subroutine being traced; + emitted verbatim into the trace string for grep-ability. + ctrl_entries : iterable of HostVarEntry + Control-variable dummies in the subroutine signature (any order; + the helper preserves it). + indent : str + Leading whitespace for the ``if`` and continuation lines. + instance_local : str, optional + Local name of ``instance_number`` when it is not already in + *ctrl_entries* (some lifecycle routines pass it separately). + extra_in_names : iterable of str, optional + Extra dummy local names to include unconditionally (treated as + ``intent(in)`` integers). Used for routines whose signatures + carry non-control intent(in) dummies that the compiler may also + flag as unused (e.g. ``number_of_instances`` in state_alloc). + + Returns + ------- + list of str + Lines forming a single ``if (trace) write(error_unit, *) ...`` + continuation block. Empty when there is nothing to print. + """ + from generator.group_cap import _ctrl_intent_for # avoid import cycle at module load + + # Build the (local_name, is_character) list in signature order. + items: List = [] + seen = set() + for entry in ctrl_entries: + if _ctrl_intent_for(entry.standard_name) == 'out': + continue + if entry.local_name in seen: + continue + seen.add(entry.local_name) + is_char = entry.type.strip().lower() == 'character' + items.append((entry.local_name, is_char)) + if instance_local and instance_local not in seen: + seen.add(instance_local) + items.append((instance_local, False)) + if extra_in_names: + for name in extra_in_names: + if name not in seen: + seen.add(name) + items.append((name, False)) + + if not items: + return [] + + # Build a per-call format: one ``a`` for the trace name, then for each + # item one ``a`` for the ``' label='`` literal plus ``a`` (character) + # or ``1x,i0`` (integer) for the value. + fmt_parts: List[str] = ['a'] + for (_, is_char) in items: + fmt_parts.append('a') + fmt_parts.append('a' if is_char else '1x,i0') + fmt = "'({})'".format(','.join(fmt_parts)) + + lines: List[str] = [] + lines.append('{}if (trace) write(error_unit, {}) &'.format(indent, fmt)) + lines.append("{} 'CCPP TRACE {}:', &".format(indent, sub_name)) + for i, (local_name, is_char) in enumerate(items): + expr = 'trim({})'.format(local_name) if is_char else local_name + sep = ', &' if i < len(items) - 1 else '' + lines.append( + "{} ' {}=', {}{}".format(indent, local_name, expr, sep) + ) + return lines diff --git a/capgen/metadata/__init__.py b/capgen/metadata/__init__.py new file mode 100644 index 00000000..f84ce9a7 --- /dev/null +++ b/capgen/metadata/__init__.py @@ -0,0 +1 @@ +"""Metadata parsing and variable resolution for ccpp-capgen.""" diff --git a/capgen/metadata/auto_clone_constituents.py b/capgen/metadata/auto_clone_constituents.py new file mode 100644 index 00000000..1e0a1233 --- /dev/null +++ b/capgen/metadata/auto_clone_constituents.py @@ -0,0 +1,243 @@ +"""TRANSIENT shim — original-capgen auto-clone-static-constituent path. + +The legacy ccpp-prebuild / original-capgen toolchain auto-registered +every ``is_constituent`` scheme arg by lifting its metadata properties +(``long_name``, ``diagnostic_name``, ``units``, ``default_value``, …) +into a synthetic ``%instantiate(...)`` call emitted into the generated +host cap. Capgen deliberately dropped that path in favour of +explicit registration (``host_constituents`` host arg + register-phase +``ccpp_constituent_properties_t(:)`` scheme args). + +This module re-enables the legacy auto-clone path as an opt-in shim +(``--legacy-auto-clone-constituents`` on the capgen CLI). It exists +for legacy host models — notably CAM-SIMA — that drive original capgen +heavily today and have not yet migrated to explicit registration. + +When enabled, the shim: + +* extends :data:`MetaVar._KNOWN_ATTRS` (in ``metadata/metadata_table.py``) + with four legacy attributes that the strict-mode parser otherwise + rejects: ``default_value``, ``min_value``, ``water_species``, + ``mixing_ratio_type``; +* enables :class:`generator.suite_resolver.SuiteResolution.auto_cloned_constituents` + collection — every ``is_constituent`` consumer whose ``std_name`` has + no register-phase source is recorded with its metadata snapshot; +* drives :func:`generator.suite_cap._register_lines` to emit synthesised + ``%instantiate(...)`` calls into the per-suite dynamic-constituents + buffer, mirroring what a hand-written register-phase scheme would do. + +## Single-instance constraint + +The legacy code paths these models came from never supported multiple +in-memory host instances. The shim follows the same restriction: when +enabled, the host metadata MUST NOT declare both ``instance_number`` +and ``number_of_instances`` (the capgen multi-instance opt-in pair +— see :data:`metadata.registered_dimensions.SCALAR_INDEX_DIMS`). The +gate is enforced in :func:`require_single_instance_host`, called from +:mod:`ccpp_capgen` after host metadata parse. Code paths under the +shim assume ``instance_number`` is the literal ``1``. + +## Self-contained for clean removal + +Every touchpoint in the rest of the codebase is tagged +``# auto-clone-constituents:``. Removing the feature is: + +1. Delete ``metadata/auto_clone_constituents.py`` +2. Delete ``unit-tests/test_auto_clone_constituents.py`` +3. Delete the sample fixture files (search ``auto_clone`` under + ``unit-tests/sample_files``) +4. ``grep -rn 'auto-clone-constituents\\|--legacy-auto-clone-constituents' .`` + and remove every remaining touchpoint (each is a 1-5 line snippet + marked with a ``# auto-clone-constituents:`` comment). + +Every hook is a no-op when the mode is not enabled — the shim has zero +impact on default capgen workflows. + +Examples +-------- +>>> from metadata import auto_clone_constituents +>>> auto_clone_constituents.is_enabled() +False +>>> auto_clone_constituents.extra_known_attrs() +frozenset() + +When enabled, four legacy attrs become recognised: + +>>> import io, logging +>>> logger = logging.getLogger('auto_clone_doctest') +>>> auto_clone_constituents.enable(logger, _stream=io.StringIO()) +>>> auto_clone_constituents.is_enabled() +True +>>> sorted(auto_clone_constituents.extra_known_attrs()) +['default_value', 'min_value', 'mixing_ratio_type', 'water_species'] +>>> auto_clone_constituents.disable() +>>> auto_clone_constituents.is_enabled() +False +""" + +from __future__ import annotations + +import sys +from typing import FrozenSet, Optional, TextIO + + +# ---------------------------------------------------------------------- +# Legacy metadata attributes the shim accepts on scheme args. +# +# Mapped to the matching kwargs of ``ccp_instantiate`` in +# ``src/ccpp_constituent_prop_mod.F90``: +# +# * default_value -> default_value (real, kind_phys) +# * min_value -> min_value (real, kind_phys) +# * water_species -> water_species (logical) +# * mixing_ratio_type -> mixing_ratio_type (character) +# +# The remaining %instantiate kwargs (std_name, long_name, diag_name, +# units, vertical_dim, advected, molar_mass) are already accepted by +# the strict-mode parser, just under canonical capgen names. +# ---------------------------------------------------------------------- +_EXTRA_KNOWN_ATTRS: FrozenSet[str] = frozenset({ + 'default_value', + 'min_value', + 'water_species', + 'mixing_ratio_type', +}) + + +# Process-level on/off flag. Mirrors the model used by +# :mod:`metadata.legacy_compat` and :mod:`metadata.dim_aliases`. +_ENABLED: bool = False + + +def enable(logger=None, _stream: Optional[TextIO] = None) -> None: + """Turn the auto-clone shim on and emit a bold warning banner. + + Idempotent: a second call is a no-op (no double warning). + + Parameters + ---------- + logger : logging.Logger, optional + Logger to emit a ``WARNING``-level companion message on. + _stream : file-like, optional + Override for the banner destination (used by tests to capture + output). ``None`` (default) means ``sys.stderr``. + """ + global _ENABLED + if _ENABLED: + return + _ENABLED = True + + stream = _stream if _stream is not None else sys.stderr + border_width = 70 + border = '*' * border_width + _content_width = border_width - len('*** ') - len(' ***') + + def _pad(s: str) -> str: + """Format *s* as a banner row, left-padded to the border width.""" + return '*** {:<{w}} ***'.format(s[:_content_width], w=_content_width) + + banner_lines = ['', border, + _pad('WARNING: LEGACY AUTO-CLONE-CONSTITUENTS ENABLED'), + _pad('')] + banner_lines += [ + _pad('Every is_constituent scheme arg (advected /'), + _pad('constituent / molar_mass) without a register-phase'), + _pad('source will be auto-registered into the per-suite'), + _pad('dynamic-constituents buffer from its scheme'), + _pad('metadata. Four legacy attributes become accepted'), + _pad('on scheme args:'), + _pad(''), + ] + for name in sorted(_EXTRA_KNOWN_ATTRS): + banner_lines.append(_pad(" '{}'".format(name))) + banner_lines += [ + _pad(''), + _pad('The host metadata MUST NOT declare instance_number'), + _pad('/ number_of_instances (single-instance only).'), + _pad(''), + _pad('This is a TRANSIENT shim for legacy hosts that have'), + _pad('not migrated to explicit registration. It WILL BE'), + _pad('REMOVED in a future capgen release.'), + border, + '', + ] + stream.write('\n'.join(banner_lines) + '\n') + try: + stream.flush() + except Exception: # pylint: disable=broad-except + pass + + if logger is not None: + logger.warning( + "Legacy auto-clone-constituents enabled: is_constituent " + "scheme args without an explicit register-phase source " + "will be auto-registered from their metadata; legacy " + "attributes %s become accepted on scheme args; host must " + "be single-instance. This shim is transient and will be " + "removed.", + ', '.join("'{}'".format(n) for n in sorted(_EXTRA_KNOWN_ATTRS)), + ) + + +def disable() -> None: + """Turn the auto-clone shim off. Intended for tests and library + users that wrap a generator invocation.""" + global _ENABLED + _ENABLED = False + + +def is_enabled() -> bool: + """Return ``True`` iff the auto-clone shim is enabled in this + process.""" + return _ENABLED + + +def extra_known_attrs() -> FrozenSet[str]: + """Return the legacy scheme-arg attribute names that the shim adds + to :data:`MetaVar._KNOWN_ATTRS` when enabled. + + When the shim is disabled, returns an empty frozenset so that the + parser's strict-mode "unknown attribute" rejection fires on these + names just as it would for any other unrecognised key. + """ + if not _ENABLED: + return frozenset() + return _EXTRA_KNOWN_ATTRS + + +def require_single_instance_host(host_dict) -> None: + """Raise :class:`CCPPError` if the host declares the multi-instance + pair (``instance_number`` + ``number_of_instances``) while the shim + is enabled. + + The auto-clone path's emitted code paths assume the literal ``1`` + for every per-instance subscript. Multi-instance support is not in + scope for this transient shim — legacy hosts that need this path + were always single-instance. Called from + :mod:`ccpp_capgen` after the host metadata has been flattened. + + No-op when the shim is disabled. Accepts the resolved + ``host_dict`` flat mapping (or any container with ``__contains__``) + so it can be wired in wherever the host has been parsed. + """ + if not _ENABLED: + return + # Lazy import to keep the shim free of generator dependencies. + from metadata.parse_tools import CCPPError # noqa: E402 + has_inst = 'instance_number' in host_dict + has_ninst = 'number_of_instances' in host_dict + if has_inst or has_ninst: + raise CCPPError( + "--legacy-auto-clone-constituents is single-instance only " + "but the host metadata declares " + "{found}. Either remove the multi-instance pair from the " + "host metadata or drop the --legacy-auto-clone-constituents " + "flag.".format( + found=' and '.join( + name for name, present in ( + ('instance_number', has_inst), + ('number_of_instances', has_ninst), + ) if present + ) + ) + ) diff --git a/capgen/metadata/dim_aliases.py b/capgen/metadata/dim_aliases.py new file mode 100644 index 00000000..faa22187 --- /dev/null +++ b/capgen/metadata/dim_aliases.py @@ -0,0 +1,196 @@ +"""TRANSIENT GFS-physics compatibility shim for equivalent dim names. + +A handful of CCPP-physics scheme groups (notably GFS radiation and +GFS chemistry / aerosol composition) declare array dimensions using +standard names that are *physically* the vertical layer dimension but +spelled differently because the legacy code path carries the historical +name around for clarity (e.g. ``adjusted_vertical_layer_dimension_for_radiation`` +on the radiation side, ``vertical_composition_dimension`` on the +composition side). + +These names cannot simply be substituted for ``vertical_layer_dimension`` +at parse time — each one is also exposed by some hosts (and consumed by +some schemes) as a standalone scalar control variable, so the +:mod:`metadata.legacy_compat` style of name rewriting would erase a +distinct variable. Instead we want them treated as *equivalent only at +the point where two metadata dimension entries (host side and scheme +side) are compared for identity*. + +This module provides an opt-in shim (``--gfs-dim-aliases`` on the +capgen / ccpp_validator CLI) that collapses each member of an alias +group to a single canonical representative when ``_canonical_dim`` +prepares a dimension entry for the strict identity comparison in +:func:`generator.suite_resolver._check_compat`. Every other consumer +keeps the original name verbatim. + +This module is **deliberately self-contained** so the workaround can +be undone with a clean delete. Removing the feature is: + +1. Delete ``metadata/dim_aliases.py`` +2. Delete ``unit-tests/test_dim_aliases.py`` +3. ``grep -rn 'gfs-dim-aliases\\|dim_aliases\\|--gfs-dim-aliases' .`` and + remove every remaining touchpoint (each is a 1-3 line snippet + marked with a ``# dim-aliases:`` comment). + +Every hook in the rest of the codebase is a no-op when the mode is +not enabled, so the shim has zero impact on default workflows. + +Examples +-------- +>>> from metadata import dim_aliases +>>> dim_aliases.is_enabled() +False +>>> dim_aliases.canonical('adjusted_vertical_layer_dimension_for_radiation') +'adjusted_vertical_layer_dimension_for_radiation' + +When enabled, each alias collapses to its group representative: + +>>> import io, logging +>>> logger = logging.getLogger('dim_aliases_doctest') +>>> dim_aliases.enable(logger, _stream=io.StringIO()) +>>> dim_aliases.is_enabled() +True +>>> dim_aliases.canonical('adjusted_vertical_layer_dimension_for_radiation') +'vertical_layer_dimension' +>>> dim_aliases.canonical('vertical_composition_dimension') +'vertical_layer_dimension' +>>> dim_aliases.canonical('air_temperature') +'air_temperature' +>>> dim_aliases.disable() +>>> dim_aliases.is_enabled() +False +""" + +from __future__ import annotations + +import sys +from typing import Dict, Optional, TextIO + + +# ---------------------------------------------------------------------- +# Alias-member -> canonical representative. +# +# Keep this list short and audited. Each entry is a deliberate +# decision that a name is physically the same axis as the +# representative *for the purpose of host/scheme dim comparison*. +# The names remain distinct as standalone variables everywhere else. +# ---------------------------------------------------------------------- +_DIM_ALIAS_MAP: Dict[str, str] = { + # GFS radiation carries a separately-named "adjusted" vertical + # layer dimension that, in practice, is the same axis as + # vertical_layer_dimension. Collapse for the per-position dim + # identity check only. + 'adjusted_vertical_layer_dimension_for_radiation': + 'vertical_layer_dimension', + # GFS chemistry / aerosol composition uses + # vertical_composition_dimension where the layer count is meant. + 'vertical_composition_dimension': + 'vertical_layer_dimension', +} + + +# Process-level on/off flag. Module state is intentional: a single +# CLI invocation is the natural unit, and threading the flag through +# every parse call would bloat the API. Tests must use the +# ``disable()`` helper to restore the default between cases. +_ENABLED: bool = False + + +def enable(logger=None, _stream: Optional[TextIO] = None) -> None: + """Turn the GFS dim-aliases shim on and emit a bold warning banner. + + The warning goes to *_stream* (defaults to ``sys.stderr``) and is + also logged at WARNING level on *logger* (if supplied) so that + downstream consumers of the logger see it. + + Idempotent: a second call is a no-op (no double warning). + + Parameters + ---------- + logger : logging.Logger, optional + Logger to emit a ``WARNING``-level companion message on. + _stream : file-like, optional + Override for the banner destination (used by tests to capture + output). ``None`` (default) means ``sys.stderr``. + """ + global _ENABLED + if _ENABLED: + return + _ENABLED = True + + stream = _stream if _stream is not None else sys.stderr + border_width = 70 + border = '*' * border_width + _content_width = border_width - len('*** ') - len(' ***') + + def _pad(s: str) -> str: + """Format *s* as a banner row, left-padded to the border width.""" + return '*** {:<{w}} ***'.format(s[:_content_width], w=_content_width) + + banner_lines = ['', border, _pad('WARNING: GFS DIM-ALIASES ENABLED'), + _pad('')] + banner_lines += [ + _pad('The following dimension standard names will be'), + _pad('treated as equivalent to their canonical axis ONLY'), + _pad('during host/scheme dim-position comparison:'), + _pad(''), + ] + for alias, canon in sorted(_DIM_ALIAS_MAP.items()): + banner_lines.append(_pad(" '{}' => '{}'".format(alias, canon))) + banner_lines += [ + _pad(''), + _pad('Variables keep their original names everywhere'), + _pad('else. This is a TRANSIENT GFS-physics shim and'), + _pad('WILL BE REMOVED in a future capgen release.'), + border, + '', + ] + stream.write('\n'.join(banner_lines) + '\n') + try: + stream.flush() + except Exception: # pylint: disable=broad-except + pass + + if logger is not None: + pair_str = ', '.join( + "'{}' => '{}'".format(alias, canon) + for alias, canon in sorted(_DIM_ALIAS_MAP.items()) + ) + logger.warning( + "GFS dim-aliases enabled: the following dimension names " + "will compare equal to their canonical axis: %s. This " + "shim is transient and will be removed.", + pair_str, + ) + + +def disable() -> None: + """Turn the dim-aliases shim off. Intended for tests and library + users that wrap a generator invocation.""" + global _ENABLED + _ENABLED = False + + +def is_enabled() -> bool: + """Return ``True`` iff the dim-aliases shim is enabled in this + process.""" + return _ENABLED + + +def canonical(name: str) -> str: + """Return the canonical representative for *name*, or *name* + unchanged. + + When the shim is **disabled** this is a strict identity — the + aliased names compare distinct, just as they would in a default + capgen workflow. When the shim is **enabled** every entry in + :data:`_DIM_ALIAS_MAP` collapses to its representative; everything + else passes through unchanged. + + Intended for use by :func:`generator.suite_resolver._canonical_dim` + on the *upper bound* of a dimension entry; no other call site + should consult this function. + """ + if not _ENABLED: + return name + return _DIM_ALIAS_MAP.get(name, name) diff --git a/capgen/metadata/legacy_compat.py b/capgen/metadata/legacy_compat.py new file mode 100644 index 00000000..059a81f4 --- /dev/null +++ b/capgen/metadata/legacy_compat.py @@ -0,0 +1,192 @@ +"""TRANSIENT compatibility shim for legacy CCPP standard names. + +The original ccpp-prebuild + ccpp-capgen toolchain used the standard +name ``horizontal_loop_extent`` where capgen uses +``horizontal_dimension``. This module provides an opt-in shim +(``--legacy-mode`` on the capgen / ccpp_validator CLI) that +silently rewrites legacy names to their canonical equivalents at +metadata parse time so the rest of the toolchain only ever sees the +canonical names. + +This module is **deliberately self-contained** so the migration can +be undone with a clean delete. Removing the feature is: + +1. Delete ``metadata/legacy_compat.py`` +2. Delete ``tests/test_legacy_compat.py`` +3. ``grep -rn 'legacy-compat\\|legacy_compat\\|--legacy-mode' .`` and + remove every remaining touchpoint (each is a 1-3 line snippet + marked with a ``# legacy-compat:`` comment). + +Every hook in the rest of the codebase is a no-op when the mode is +not enabled, so the shim has zero impact on non-legacy workflows. + +Examples +-------- +>>> from metadata import legacy_compat +>>> legacy_compat.is_enabled() +False +>>> legacy_compat.translate('horizontal_loop_extent') +'horizontal_loop_extent' +>>> legacy_compat.translate('air_temperature') +'air_temperature' + +When enabled, legacy names are rewritten: + +>>> import io, logging +>>> logger = logging.getLogger('legacy_compat_doctest') +>>> legacy_compat.enable(logger, _stream=io.StringIO()) +>>> legacy_compat.is_enabled() +True +>>> legacy_compat.translate('horizontal_loop_extent') +'horizontal_dimension' +>>> legacy_compat.translate('air_temperature') +'air_temperature' +>>> legacy_compat.disable() +>>> legacy_compat.is_enabled() +False +""" + +from __future__ import annotations + +import sys +from typing import Dict, Optional, TextIO + + +# ---------------------------------------------------------------------- +# Legacy → canonical name map. +# +# Keep this short and audited. Every entry is a deliberate decision +# that a legacy name has a single, unambiguous canonical replacement. +# ---------------------------------------------------------------------- +_LEGACY_NAME_MAP: Dict[str, str] = { + # ccpp-prebuild / original ccpp-capgen used ``horizontal_loop_extent`` + # in scheme metadata where capgen uses ``horizontal_dimension``. + 'horizontal_loop_extent': 'horizontal_dimension', + + # Legacy CCPP-physics hosts (and SCM 17p8 in particular) sized + # per-thread DDT containers by ``number_of_openmp_threads``; the + # capgen convention is ``number_of_threads`` (matching the + # ``thread_number`` control variable name). Aliasing here lets the + # host metadata flow through unchanged; once hosts have migrated, + # drop this entry. + 'number_of_openmp_threads': 'number_of_threads', +} + + +# Process-level on/off flag. Module state is intentional: a single +# CLI invocation is the natural unit, and threading the flag through +# every parse call would bloat the API. Tests must use the +# ``disable()`` helper (or the ``legacy_mode_disabled`` context +# manager) to restore the default between cases. +_ENABLED: bool = False + + +def enable(logger=None, _stream: Optional[TextIO] = None) -> None: + """Turn legacy mode on and emit a single bold warning banner. + + The warning goes to *_stream* (defaults to ``sys.stderr``) and is + also logged at WARNING level on *logger* (if supplied) so that + downstream consumers of the logger see it. + + Idempotent: a second call is a no-op (no double warning). + + Parameters + ---------- + logger : logging.Logger, optional + Logger to emit a ``WARNING``-level companion message on. + _stream : file-like, optional + Override for the banner destination (used by tests to capture + output). ``None`` (default) means ``sys.stderr``. + """ + global _ENABLED + if _ENABLED: + return + _ENABLED = True + + stream = _stream if _stream is not None else sys.stderr + border_width = 70 + border = '*' * border_width + # Content width between the leading ``*** `` and trailing ` ***``. + _content_width = border_width - len('*** ') - len(' ***') + + def _pad(s: str) -> str: + """Format *s* as a banner row, left-padded to the border width.""" + return '*** {:<{w}} ***'.format(s[:_content_width], w=_content_width) + + banner_lines = ['', border, _pad('WARNING: LEGACY-MODE ENABLED'), + _pad('')] + if len(_LEGACY_NAME_MAP) == 1: + # Singular phrasing reads better when there's only one pair. + old, new = next(iter(_LEGACY_NAME_MAP.items())) + banner_lines += [ + _pad('Metadata using the deprecated standard name'), + _pad(" '{}'".format(old)), + _pad('will be silently rewritten to'), + _pad(" '{}'".format(new)), + _pad('at parse time.'), + ] + else: + banner_lines += [ + _pad('Metadata using any of these deprecated standard names'), + _pad('will be silently rewritten at parse time:'), + _pad(''), + ] + for old, new in sorted(_LEGACY_NAME_MAP.items()): + banner_lines.append(_pad(" '{}' -> '{}'".format(old, new))) + banner_lines += [ + _pad(''), + _pad('This is a TRANSIENT migration shim. Update your'), + _pad('metadata to use the canonical names; legacy mode'), + _pad('WILL BE REMOVED in a future capgen release.'), + border, + '', + ] + stream.write('\n'.join(banner_lines) + '\n') + try: + stream.flush() + except Exception: # pylint: disable=broad-except + pass + + if logger is not None: + pair_str = ', '.join( + "'{}' -> '{}'".format(old, new) + for old, new in sorted(_LEGACY_NAME_MAP.items()) + ) + logger.warning( + "Legacy mode enabled: the following deprecated standard " + "names will be rewritten in metadata at parse time: %s. " + "This shim is transient and will be removed.", + pair_str, + ) + + +def disable() -> None: + """Turn legacy mode off. Intended for tests and library users that + wrap a generator invocation.""" + global _ENABLED + _ENABLED = False + + +def is_enabled() -> bool: + """Return ``True`` iff legacy mode has been enabled in this process.""" + return _ENABLED + + +def translate(name: str) -> str: + """Return the canonical replacement for *name*, or *name* unchanged. + + When legacy mode is **disabled** this is a strict identity — even + a legacy name like ``horizontal_loop_extent`` is returned as-is so + downstream parsers reject it just as they would in non-legacy + workflows. When legacy mode is **enabled** the entries in + :data:`_LEGACY_NAME_MAP` are rewritten and everything else passes + through unchanged. + + The function is tolerant of any input that lookup-by-string is + valid for (``str``). Callers may pre-lowercase the input — the + map keys are already lowercase to match capgen's + case-insensitive standard-name convention. + """ + if not _ENABLED: + return name + return _LEGACY_NAME_MAP.get(name, name) diff --git a/capgen/metadata/metadata_table.py b/capgen/metadata/metadata_table.py new file mode 100644 index 00000000..995ca7c6 --- /dev/null +++ b/capgen/metadata/metadata_table.py @@ -0,0 +1,1472 @@ +#!/usr/bin/env python3 + +"""Metadata table parser for ccpp-capgen. + +Each ``.meta`` file contains one or more CCPP metadata tables. Every table +begins with a ``[ccpp-table-properties]`` header, followed by one or more +``[ccpp-arg-table]`` section headers, each followed by variable blocks. + +Supported table types +--------------------- +``scheme`` + Physics scheme subroutine interfaces. One ``[ccpp-arg-table]`` section + per phase (``register``, ``init``, ``timestep_init``, ``run``, + ``timestep_final``, ``final``). All variables carry an ``intent`` + attribute. + +``host`` + Host-model module data (replaces the old ``module`` type — using + ``type = module`` is a hard error in this generator). + +``control`` + Framework control variables passed explicitly as subroutine arguments + (e.g. ``horizontal_loop_begin``, ``ccpp_error_code``). + +``ddt`` + Derived data type (DDT) structural definition. Describes field layout; + no instance information. Instances are declared as variables inside a + ``host`` table. + +``suite`` + Generator-written tables for suite-owned interstitial data. Never + hand-authored. + +Format reference +---------------- +:: + + [ccpp-table-properties] + name = + type = + + [ccpp-arg-table] + name = # scheme: _; others: same as table_name + type = # must match [ccpp-table-properties] type + + [ local_name ] + standard_name = + long_name = # optional + units = # optional; defaults to 'none' + dimensions = (, , ...) # () for scalar + type = + kind = # optional + intent = in | out | inout # required for scheme vars + optional = True | False # default False + active = # optional; uses standard names + protected = True | False # default False + allocatable = True | False # default False + diagnostic_name = # optional; host-tooling hint + diagnostic_name_fixed = # optional; mutually exclusive + # with diagnostic_name + +Multiple properties may appear on one line, separated by ``|``. + +DDT instance entries in a ``host`` table use a DDT type name as ``type``, and +may declare ``dimensions = (number_of_instances)`` for array instances. + +External (non-CCPP) DDT types use the syntax:: + + type = external:: + +e.g. ``type = external:mpi_f08:mpi_comm``. +""" + +import os +import re +from typing import Dict, List, Optional, Tuple + +# legacy-compat: transient migration shim (delete with the rest of +# the legacy_compat touchpoints — grep for ``legacy-compat``). +from . import legacy_compat +# auto-clone-constituents: transient shim (delete with the rest of +# the auto-clone-constituents touchpoints — grep for +# ``auto-clone-constituents``). +from . import auto_clone_constituents +from .parse_tools import ( + CCPPError, + ParseContext, + ParseSyntaxError, + check_cf_standard_name, + check_units, + check_dimensions, + check_diagnostic_fixed, + check_diagnostic_id, + check_fortran_id, + check_fortran_ref, + check_fortran_intrinsic, + check_molar_mass, + # auto-clone-constituents: legacy-shim checkers. + check_default_value, + check_min_value, + check_water_species, + check_mixing_ratio_type, +) + +######################################################################## +# Module-level constants +######################################################################## + +#: All table type values accepted by the parser. +VALID_TABLE_TYPES = frozenset({'scheme', 'host', 'control', 'suite', 'ddt'}) + +#: Table types that have exactly one ``[ccpp-arg-table]`` section per table. +SINGLETON_TABLE_TYPES = frozenset({'host', 'control', 'suite', 'ddt'}) + +#: The scheme table type (the only type with multiple sections). +SCHEME_TABLE_TYPE = 'scheme' + +#: Valid scheme phase suffixes. ``finalize`` is renamed to ``final``; using +#: ``_finalize`` in a section name is a hard error. +VALID_SCHEME_PHASES = frozenset({ + 'register', 'init', 'timestep_init', 'run', 'timestep_final', 'final' +}) + +#: Valid intent values for scheme variables. +VALID_INTENTS = frozenset({'in', 'out', 'inout'}) + +#: Maximum allowed length for a Fortran identifier (F2018 §6.1.1). +FORTRAN_MAX_IDENT_LEN = 63 + +#: Regex for a bare section header ``[ name ]``. +_VAR_HEADER_RE = re.compile(r"^\[\s*(\S+)\s*\]\s*$") + +#: Regex for the reserved section keywords. +_TABLE_PROPS_HDR = '[ccpp-table-properties]' +_ARG_TABLE_HDR = '[ccpp-arg-table]' + +#: Lines that are blank or start with a comment character. +_BLANK_RE = re.compile(r"^\s*([#;].*)?$") + +#: ``external:module:typename`` DDT type syntax. +_EXTERNAL_TYPE_RE = re.compile( + r'^external\s*:\s*([A-Za-z][A-Za-z0-9_]*)\s*:\s*([A-Za-z][A-Za-z0-9_]*)$', + re.IGNORECASE, +) + +#: ``kind_spec`` value in ``[ccpp-table-properties]``. Two accepted forms: +#: +#: :=>spec -- explicit CCPP-visible kind name +#: : -- shorthand; kind_name defaults to spec +#: +#: Captured groups: (module, kind_name_or_None, spec). When the second +#: group is None the caller substitutes ``spec`` for ``kind_name``. +_KIND_SPEC_RE = re.compile( + r'^\s*([A-Za-z][A-Za-z0-9_]*)\s*:\s*' + r'(?:([A-Za-z][A-Za-z0-9_]*)\s*=>\s*)?' + r'([A-Za-z][A-Za-z0-9_]*)\s*$' +) + +######################################################################## +# Helper functions +######################################################################## + +def _is_blank(line: str) -> bool: + """Return True for blank lines and comment-only lines. + + >>> _is_blank('') + True + >>> _is_blank(' ') + True + >>> _is_blank('# comment') + True + >>> _is_blank('; comment') + True + >>> _is_blank(' name = foo') + False + """ + return _BLANK_RE.match(line) is not None + + +def _strip_inline_comment(line: str) -> str: + """Drop a trailing ``# ...`` comment from a metadata line. + + Metadata files use ``#`` (and ``;`` at column 0) as comment markers. + A ``#`` anywhere in a line — not just at column 0 — starts a comment + that runs to the end of the line; the parser discards it before any + further processing. No escape mechanism: ``#`` is not a legitimate + character in any metadata value (units, kinds, identifiers, dim + lists, Fortran conditional expressions). + + Trailing whitespace left behind by the strip is also removed so that + section/variable headers like ``[ name ]`` and key=value lines parse + cleanly with their existing regexes. + + >>> _strip_inline_comment('dimensions = () # (nap_indices)') + 'dimensions = ()' + >>> _strip_inline_comment('[ ap_indices ] # legacy slot') + '[ ap_indices ]' + >>> _strip_inline_comment('# whole-line comment') + '' + >>> _strip_inline_comment('plain line with no comment') + 'plain line with no comment' + >>> _strip_inline_comment('') + '' + """ + idx = line.find('#') + if idx < 0: + return line + return line[:idx].rstrip() + + +def _parse_bool(value: str, context: ParseContext) -> bool: + """Parse a Fortran/Python boolean string to a Python bool. + + Accepts ``True``/``False`` (case-insensitive). + + >>> from metadata.parse_tools import ParseContext + >>> ctx = ParseContext(0, 'test.meta') + >>> _parse_bool('True', ctx) + True + >>> _parse_bool('false', ctx) + False + >>> _parse_bool('.true.', ctx) + True + >>> _parse_bool('.false.', ctx) + False + """ + normalized = value.strip().lower() + if normalized in ('true', '.true.', 't', '1'): + return True + if normalized in ('false', '.false.', 'f', '0'): + return False + raise CCPPError( + "Invalid boolean '{}', at {}".format(value, context) + ) + + +def _parse_kind_spec_value( + value: str, context: ParseContext, +) -> Tuple[str, str, str]: + """Parse one ``kind_spec`` value into ``(kind_name, module, spec)``. + + Accepted syntax:: + + :=>spec # explicit CCPP-visible kind name + : # kind_name defaults to spec + + All three components must be valid Fortran identifiers. Whitespace + around the separators is tolerated. + + >>> from metadata.parse_tools import ParseContext + >>> ctx = ParseContext(0, 'test.meta') + >>> _parse_kind_spec_value('temp_kinds:kind_temp=>temp_r8', ctx) + ('kind_temp', 'temp_kinds', 'temp_r8') + >>> _parse_kind_spec_value('host_kinds:kind_r8', ctx) + ('kind_r8', 'host_kinds', 'kind_r8') + >>> _parse_kind_spec_value(' temp_kinds : kind_temp => temp_r8 ', ctx) + ('kind_temp', 'temp_kinds', 'temp_r8') + >>> _parse_kind_spec_value('not_a_kind_spec', ctx) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + metadata.parse_tools.parse_source.CCPPError: Malformed kind_spec ... + """ + match = _KIND_SPEC_RE.match(value) + if match is None: + raise CCPPError( + "Malformed kind_spec '{}', at {}: expected " + ":=>spec or :".format( + value, context + ) + ) + module = match.group(1) + kind_name = match.group(2) + spec = match.group(3) + if kind_name is None: + kind_name = spec + return kind_name, module, spec + + +def _parse_dimensions(value: str, context: ParseContext) -> List[str]: + """Parse a dimension list ``(d1, d2, ...)`` into a Python list. + + The empty list ``[]`` represents a scalar (``dimensions = ()``). + + >>> from metadata.parse_tools import ParseContext + >>> ctx = ParseContext(0, 'test.meta') + >>> _parse_dimensions('()', ctx) + [] + >>> _parse_dimensions('(horizontal_dimension)', ctx) + ['horizontal_dimension'] + >>> _parse_dimensions('(horizontal_dimension, vertical_layer_dimension)', ctx) + ['horizontal_dimension', 'vertical_layer_dimension'] + + CCPP standard names are case-insensitive; mixed-case spellings in + the metadata are normalised to lower-case so downstream lookups + against ``host_dict`` (which stores std names lower-cased per + ``check_cf_standard_name``) succeed: + + >>> _parse_dimensions('(number_of_aerosol_tracers_MG)', ctx) + ['number_of_aerosol_tracers_mg'] + >>> _parse_dimensions('(ccpp_constant_one:Vertical_Layer_Dimension)', ctx) + ['ccpp_constant_one:vertical_layer_dimension'] + """ + stripped = value.strip() + if not (stripped.startswith('(') and stripped.endswith(')')): + raise ParseSyntaxError( + "dimensions value (must be parenthesised list)", token=value, + context=context + ) + inner = stripped[1:-1].strip() + if not inner: + return [] + parts = [p.strip() for p in inner.split(',')] + normalised: List[str] = [] + for part in parts: + if not part: + raise ParseSyntaxError( + "empty dimension entry in '{}'".format(value), + context=context + ) + check_dimensions([part], None, error=True) + # Lowercase every non-integer token so the resolver's + # host_dict lookups succeed regardless of the user's metadata + # casing. Range form ``lower:upper`` lowercases each half; + # integer literals are unchanged by ``.lower()``. + # + # legacy-compat: after lowercasing, run each token through the + # legacy translator so ``horizontal_loop_extent`` (etc.) is + # rewritten to its canonical name. No-op when legacy mode is + # disabled. + normalised.append(':'.join( + legacy_compat.translate(t.strip().lower()) + for t in part.split(':') + )) + return normalised + + +def _check_var_type(value: str, context: ParseContext) -> str: + """Validate and normalise a variable ``type`` attribute. + + Accepts: + * Fortran intrinsic types (case-insensitive, returned as given). + * ``type()`` DDT references (returned as given). + * Plain ```` DDT type names (returned as given). + * ``external::`` for non-CCPP types. + + >>> from metadata.parse_tools import ParseContext + >>> ctx = ParseContext(0, 'test.meta') + >>> _check_var_type('real', ctx) + 'real' + >>> _check_var_type('integer', ctx) + 'integer' + >>> _check_var_type('gfs_statein_type', ctx) + 'gfs_statein_type' + >>> _check_var_type('external:mpi_f08:mpi_comm', ctx) + 'external:mpi_f08:mpi_comm' + """ + stripped = value.strip() + # Fortran intrinsic? + if check_fortran_intrinsic(stripped, error=False) is not None: + return stripped + # external:module:typename? + if _EXTERNAL_TYPE_RE.match(stripped): + return stripped + # type(identifier) form? + m = re.match(r'(?i)^type\s*\(\s*([A-Za-z][A-Za-z0-9_]*)\s*\)$', stripped) + if m: + return stripped + # Plain DDT name (a valid Fortran identifier)? + if check_fortran_id(stripped, None, error=False) is not None: + return stripped + raise ParseSyntaxError( + "variable type", token=value, context=context + ) + + +def _parse_config_line(line: str, context: ParseContext) -> List[Tuple[str, str]]: + """Parse one ini-format key=value line (possibly multiple pairs per line + separated by ``|``). + + Returns a list of ``(key, value)`` pairs. + + >>> from metadata.parse_tools import ParseContext + >>> ctx = ParseContext(0, 'test.meta') + >>> _parse_config_line(' name = foo ', ctx) + [('name', 'foo')] + >>> _parse_config_line('units = 1 | dimensions = ()', ctx) + [('units', '1'), ('dimensions', '()')] + >>> _parse_config_line('', ctx) + [] + """ + if _is_blank(line): + return [] + pairs = [] + for segment in line.split('|'): + parts = segment.split('=', 1) + if len(parts) != 2: + raise ParseSyntaxError( + "key=value pair", token=segment.strip(), context=context + ) + key = parts[0].strip().lower() + val = parts[1].strip() + if not key: + raise ParseSyntaxError( + "empty key in property", token=segment.strip(), context=context + ) + pairs.append((key, val)) + return pairs + + +######################################################################## +# Core data classes +######################################################################## + +class MetaVar: + """A single variable entry parsed from a CCPP metadata table. + + Attributes + ---------- + local_name : str + The Fortran local variable name (from the ``[ name ]`` header). + standard_name : str + CF-compliant standard name (lowercase). + long_name : str + Human-readable description (may be empty). + units : str + Physical units string. Defaults to ``'none'`` if the metadata + entry omits the ``units`` attribute. + dimensions : list of str + Ordered list of dimension standard names; empty list for scalars. + type : str + Fortran type string (intrinsic, DDT name, or ``external:m:t``). + kind : str + Optional Fortran kind parameter (empty string if absent). + intent : str or None + ``'in'``, ``'out'``, or ``'inout'`` for scheme variables; ``None`` + for host/ddt/control variables. + optional : bool + Whether the variable is optional (default ``False``). + active : str + Fortran conditional expression (in standard names) controlling when + the variable is present; empty string if unconditionally active. + protected : bool + If ``True``, any scheme declaring ``intent`` other than ``in`` is a + metadata error. + allocatable : bool + Whether the variable is declared with the Fortran ``allocatable`` + attribute (default ``False``). Host and scheme metadata must agree + on this flag for matching standard names. Affects code generation: + actual arguments at call sites omit explicit dimension subscripts + for allocatable variables. + diagnostic_name : str + Optional host-tooling hint: the name under which the variable is + exposed to the host model's diagnostic / history-output system. May + contain ``${process}`` or ``${scheme_name}`` substitutions. Mutually + exclusive with :attr:`diagnostic_name_fixed`. When neither attribute + is explicitly set this property defaults to :attr:`local_name`, so + downstream consumers always see a non-empty value unless + ``diagnostic_name_fixed`` was provided instead. + diagnostic_name_fixed : str + Like :attr:`diagnostic_name` but a bare Fortran identifier with no + substitutions allowed. Mutually exclusive with + :attr:`diagnostic_name`. + context : ParseContext + Source location for diagnostic messages. + + Examples + -------- + >>> from metadata.parse_tools import ParseContext + >>> ctx = ParseContext(10, 'example.meta') + >>> v = MetaVar('im', ctx) + >>> v.set_attr('standard_name', 'horizontal_dimension', ctx) + >>> v.set_attr('units', 'count', ctx) + >>> v.set_attr('dimensions', '()', ctx) + >>> v.set_attr('type', 'integer', ctx) + >>> v.set_attr('intent', 'in', ctx) + >>> v.standard_name + 'horizontal_dimension' + >>> v.intent + 'in' + >>> v.dimensions + [] + """ + + # Attributes that are boolean flags. + _BOOL_ATTRS = frozenset({'optional', 'protected', 'allocatable'}) + + # All recognised per-variable attributes. + # ``diagnostic_name`` / ``diagnostic_name_fixed`` are host-tooling hints + # propagated to ``datatable.xml``; the cap code itself does not consume + # them. The two are mutually exclusive. + _KNOWN_ATTRS = frozenset({ + 'standard_name', 'long_name', 'units', 'dimensions', + 'type', 'kind', 'intent', 'optional', 'active', 'protected', + 'allocatable', + 'diagnostic_name', 'diagnostic_name_fixed', + 'constituent', 'advected', 'molar_mass', + 'top_at_one', + }) + + # auto-clone-constituents: the legacy auto-clone shim extends the + # accepted set with four ``%instantiate``-kwarg-mapped attrs + # (default_value, min_value, water_species, mixing_ratio_type). + # ``_known_attrs()`` returns the union when the shim is enabled, + # the base set otherwise — strict mode keeps rejecting the legacy + # names with the original "Unknown variable attribute" error. + @classmethod + def _known_attrs(cls): + extra = auto_clone_constituents.extra_known_attrs() + if extra: + return cls._KNOWN_ATTRS | extra + return cls._KNOWN_ATTRS + + def __init__(self, local_name: str, context: ParseContext): + """Initialise with *local_name* from the ``[ name ]`` section header. + + The bracket header accepts either a bare Fortran identifier + (``[foo]``) or a sliced reference (``[dqdt(:,:,index_of_)]``). + The base identifier must be a valid Fortran name within the + 63-char limit; subscript tokens may be CCPP standard names + (which routinely exceed 63 chars) and are validated separately + as references later — apply the length check only to the base. + + All other attributes are set later via :meth:`set_attr`. + """ + # Step 1: validate the overall syntactic form (bare id or array + # reference) without imposing length limits on subscript tokens. + if check_fortran_ref(local_name, None, error=False, max_len=0) is None: + raise ParseSyntaxError( + "variable local name (must be a valid Fortran identifier " + "or scalar array reference)", + token=local_name, context=context + ) + # Step 2: enforce the Fortran-identifier length limit on the + # base name only (everything before the first ``(``). + paren = local_name.find('(') + base_name = local_name[:paren].strip() if paren >= 0 else local_name.strip() + if len(base_name) > FORTRAN_MAX_IDENT_LEN: + raise ParseSyntaxError( + "variable local name base '{}' is longer than the " + "Fortran identifier limit ({} chars)".format( + base_name, FORTRAN_MAX_IDENT_LEN, + ), + token=local_name, context=context + ) + self.local_name: str = local_name + self.standard_name: str = '' + self.long_name: str = '' + self.units: str = 'none' + self.dimensions: List[str] = [] + self.type: str = '' + self.kind: str = '' + self.intent: Optional[str] = None + self.optional: bool = False + self.active: str = '' + self.protected: bool = False + self.allocatable: bool = False + # Backing field for the :attr:`diagnostic_name` property. ``set_attr`` + # writes the explicitly-supplied value here; the property layers the + # ``local_name`` default on top. + self._diagnostic_name: str = '' + self.diagnostic_name_fixed: str = '' + # Constituent-related hints (scheme metadata only). A variable is + # treated as a constituent if any of ``constituent``, ``advected``, or + # ``molar_mass`` is set to a non-default value — see + # :attr:`is_constituent`. + self.constituent: bool = False + self.advected: bool = False + self.molar_mass: float = 0.0 + # ``top_at_one`` declares the vertical-axis ordering for arrays with a + # vertical dimension. ``True`` means the model top is at index 1 (k=1 + # topmost, k=nz surface); the default ``False`` means surface at 1 + # (k=1 surface, k=nz top). When a scheme and the host disagree on + # this flag the generator emits a vertical-flip transform that + # substitutes the vertical index with `` - k + 1`` on the + # host-side access expression. + self.top_at_one: bool = False + # auto-clone-constituents: legacy attrs accepted only when the + # shim is enabled. Backing fields use ``None`` as the + # "not set" sentinel so the emitter can distinguish an + # explicit value from a default (optional kwargs on + # %instantiate are only passed when explicitly set). + self.default_value: Optional[float] = None + self.min_value: Optional[float] = None + self.water_species: Optional[bool] = None + self.mixing_ratio_type: Optional[str] = None + self.context: ParseContext = context + # Track which attributes have been explicitly set (for validation). + self._set_attrs: set = set() + + # ------------------------------------------------------------------ + def set_attr(self, key: str, value: str, context: ParseContext) -> None: + """Store attribute *key* = *value* after validating the value. + + Parameters + ---------- + key : str + Lower-case attribute name. + value : str + Raw string value from the metadata file. + context : ParseContext + Source location (for error messages). + + Raises + ------ + CCPPError + On unknown attribute names or invalid values. + """ + # auto-clone-constituents: consult the dynamic union so the + # legacy attrs (default_value/min_value/water_species/ + # mixing_ratio_type) are accepted only when the shim is on. + if key not in self._known_attrs(): + raise CCPPError( + "Unknown variable attribute '{}' for '{}', at {}".format( + key, self.local_name, context + ) + ) + if key in self._set_attrs: + raise CCPPError( + "Duplicate attribute '{}' for variable '{}', at {}".format( + key, self.local_name, context + ) + ) + self._set_attrs.add(key) + + # Wrap the per-attribute validation so any CCPPError raised by a + # check_X helper (which only sees the raw value) gets enriched + # with the variable name, the attribute name, and the source + # location. Without this, a parse failure like an empty + # ``units =`` line surfaces as a bare "'' is not a valid unit" + # with no clue which file/line/variable is at fault. + try: + if key == 'standard_name': + # legacy-compat: rewrite deprecated names (e.g. + # horizontal_loop_extent → horizontal_dimension) when + # legacy mode is enabled. Applied *after* + # check_cf_standard_name (which lowercases) so mixed-case + # legacy spellings are captured. No-op otherwise. + self.standard_name = legacy_compat.translate( + check_cf_standard_name(value, None, error=True)) + elif key == 'long_name': + self.long_name = value + elif key == 'units': + self.units = check_units(value, None, error=True) + elif key == 'dimensions': + self.dimensions = _parse_dimensions(value, context) + elif key == 'type': + self.type = _check_var_type(value, context) + elif key == 'kind': + self.kind = value.strip() + elif key == 'intent': + iv = value.strip().lower() + if iv not in VALID_INTENTS: + raise CCPPError( + "Invalid intent '{}'; must be one of {}".format( + value, sorted(VALID_INTENTS), + ) + ) + self.intent = iv + elif key == 'optional': + self.optional = _parse_bool(value, context) + elif key == 'active': + # Standard names elsewhere are canonicalised to lowercase by + # check_cf_standard_name; an active expression references those + # same names, so normalise here too. Fortran is case-insensitive, + # so embedded logical operators/literals are unaffected. + self.active = value.strip().lower() + elif key == 'protected': + self.protected = _parse_bool(value, context) + elif key == 'allocatable': + self.allocatable = _parse_bool(value, context) + elif key == 'diagnostic_name': + self._diagnostic_name = check_diagnostic_id( + value.strip(), self._prop_snapshot(), error=True + ) + elif key == 'diagnostic_name_fixed': + self.diagnostic_name_fixed = check_diagnostic_fixed( + value.strip(), self._prop_snapshot(), error=True + ) + elif key == 'constituent': + self.constituent = _parse_bool(value, context) + elif key == 'advected': + self.advected = _parse_bool(value, context) + elif key == 'molar_mass': + self.molar_mass = check_molar_mass(value.strip(), None, error=True) + elif key == 'top_at_one': + self.top_at_one = _parse_bool(value, context) + # auto-clone-constituents: BEGIN legacy-shim attr setters. + # Only reachable when ``_known_attrs()`` returned the + # union, so the strict-mode unknown-attr error fires + # before we get here when the shim is off. + elif key == 'default_value': + self.default_value = check_default_value( + value.strip(), None, error=True) + elif key == 'min_value': + self.min_value = check_min_value( + value.strip(), None, error=True) + elif key == 'water_species': + self.water_species = check_water_species( + value.strip(), None, error=True) + elif key == 'mixing_ratio_type': + self.mixing_ratio_type = check_mixing_ratio_type( + value.strip(), None, error=True) + # auto-clone-constituents: END legacy-shim attr setters. + except CCPPError as exc: + # Avoid double-wrapping if the inner check already carried + # the location (some helpers do; most don't). + inner = str(exc) + location = str(context) + if location and location in inner: + raise + raise CCPPError( + "Invalid metadata for variable '{name}', attribute " + "'{key}' = '{value}', at {ctx}:\n {inner}".format( + name=self.local_name or '', + key=key, + value=value, + ctx=context, + inner=inner, + ) + ) from exc + + # ------------------------------------------------------------------ + @property + def is_constituent(self) -> bool: + """Return True iff this variable is flagged as a constituent. + + A variable is treated as a constituent when any of the three + constituent-hint attributes is set to a non-default value: + + * ``constituent = True`` + * ``advected = True`` + * ``molar_mass != 0.0`` + + Matches the rollup in the original capgen + (``scripts/metavar.py`` ``__is_constituent``). + """ + return self.constituent or self.advected or self.molar_mass != 0.0 + + # ------------------------------------------------------------------ + @property + def diagnostic_name(self) -> str: + """Effective diagnostic name. + + Returns the explicitly-set value when one was provided. Otherwise, + when ``diagnostic_name_fixed`` is also unset, falls back to + :attr:`local_name` (matching the original capgen + ``local_name_to_diag_name`` default). Returns an empty string only + when ``diagnostic_name_fixed`` was provided instead. + """ + if self._diagnostic_name: + return self._diagnostic_name + if self.diagnostic_name_fixed: + return '' + return self.local_name + + def _prop_snapshot(self) -> Dict[str, str]: + """Return a dict snapshot of attributes the diagnostic checkers may inspect. + + The ``check_diagnostic_id`` and ``check_diagnostic_fixed`` helpers + cross-validate the two diagnostic attributes and reference the + variable's local and standard names in error messages. The snapshot + reports the *explicitly-set* ``diagnostic_name`` (not the + ``local_name`` default) so the mutual-exclusion check fires only when + the author actually set it. + """ + return { + 'local_name' : self.local_name, + 'standard_name' : self.standard_name, + 'diagnostic_name' : self._diagnostic_name, + 'diagnostic_name_fixed': self.diagnostic_name_fixed, + } + + # ------------------------------------------------------------------ + def validate(self, require_intent: bool, context: ParseContext) -> None: + """Check that all required attributes are present. + + Parameters + ---------- + require_intent : bool + If ``True`` (scheme variables), ``intent`` must be set. + context : ParseContext + Source location (for error messages). + + Raises + ------ + CCPPError + If any required attribute is missing. + """ + required = {'standard_name', 'dimensions', 'type'} + if require_intent: + required.add('intent') + missing = required - self._set_attrs + if missing: + raise CCPPError( + "Variable '{}' is missing required attributes: {}, at {}".format( + self.local_name, sorted(missing), context + ) + ) + + # ------------------------------------------------------------------ + def __repr__(self) -> str: + return "MetaVar({!r}, standard_name={!r})".format( + self.local_name, self.standard_name + ) + + def is_external_ddt(self) -> bool: + """Return True if this variable's type is an external (non-CCPP) DDT.""" + return _EXTERNAL_TYPE_RE.match(self.type) is not None + + def external_ddt_module(self) -> Optional[str]: + """Return the module name for an external DDT type, or None.""" + m = _EXTERNAL_TYPE_RE.match(self.type) + return m.group(1) if m else None + + def external_ddt_typename(self) -> Optional[str]: + """Return the type name for an external DDT type, or None.""" + m = _EXTERNAL_TYPE_RE.match(self.type) + return m.group(2) if m else None + + +######################################################################## + +class MetadataSection: + """One ``[ccpp-arg-table]`` section: name, type, and variables. + + For scheme tables the section name encodes the phase: + ``_`` where *phase* is one of + ``register``, ``init``, ``timestep_init``, ``run``, + ``timestep_final``, ``final``. + + For non-scheme tables the section name equals the table name. + + Parameters + ---------- + section_name : str + The ``name`` attribute from ``[ccpp-arg-table]``. + section_type : str + The ``type`` attribute (must match the enclosing table type). + table_name : str + The enclosing table's name (used for validation and error messages). + context : ParseContext + Source location. + + Examples + -------- + >>> from metadata.parse_tools import ParseContext + >>> ctx = ParseContext(5, 'scheme.meta') + >>> sec = MetadataSection('my_scheme_run', 'scheme', 'my_scheme', ctx) + >>> sec.section_name + 'my_scheme_run' + >>> sec.phase + 'run' + >>> sec = MetadataSection('host_data', 'host', 'host_data', ctx) + >>> sec.phase is None + True + """ + + def __init__(self, section_name: str, section_type: str, + table_name: str, context: ParseContext): + if section_type not in VALID_TABLE_TYPES: + raise CCPPError( + "Section type '{}' is not a valid table type; " + "must be one of {}, at {}".format( + section_type, sorted(VALID_TABLE_TYPES), context + ) + ) + self.section_name: str = section_name + self.section_type: str = section_type + self.context: ParseContext = context + self.variables: List[MetaVar] = [] + self._std_name_index: Dict[str, MetaVar] = {} + # Derive the scheme phase from the section name (scheme only). + self._phase: Optional[str] = None + if section_type == SCHEME_TABLE_TYPE: + self._phase = self._extract_phase(section_name, table_name, context) + + @staticmethod + def _extract_phase(section_name: str, scheme_name: str, + context: ParseContext) -> str: + """Extract and validate the phase suffix from a scheme section name. + + The section name must have the form ``_``. + If the suffix is ``finalize`` (the old name), a hard error is raised + directing the user to rename it to ``final``. + """ + prefix = scheme_name + '_' + if not section_name.startswith(prefix): + raise CCPPError( + "Scheme section name '{}' does not begin with scheme name '{}', " + "at {}".format(section_name, scheme_name, context) + ) + phase = section_name[len(prefix):] + if phase == 'finalize': + raise CCPPError( + "Phase 'finalize' has been renamed to 'final'; " + "rename '{}' to '{}', at {}".format( + section_name, scheme_name + '_final', context + ) + ) + if phase not in VALID_SCHEME_PHASES: + raise CCPPError( + "Unknown scheme phase '{}' in section '{}'; " + "must be one of {}, at {}".format( + phase, section_name, sorted(VALID_SCHEME_PHASES), context + ) + ) + return phase + + @property + def phase(self) -> Optional[str]: + """The scheme phase (``'run'``, ``'init'``, etc.) or ``None``.""" + return self._phase + + def add_variable(self, var: MetaVar) -> None: + """Append *var* to this section, checking for duplicate standard names.""" + if var.standard_name in self._std_name_index: + existing = self._std_name_index[var.standard_name] + raise CCPPError( + "Duplicate standard name '{}' in section '{}': " + "first at {}, duplicate at {}".format( + var.standard_name, self.section_name, + existing.context, var.context + ) + ) + self.variables.append(var) + self._std_name_index[var.standard_name] = var + + def get_variable(self, standard_name: str) -> Optional[MetaVar]: + """Return the variable with *standard_name*, or ``None``.""" + return self._std_name_index.get(standard_name) + + def __repr__(self) -> str: + return "MetadataSection({!r}, nvars={})".format( + self.section_name, len(self.variables) + ) + + +######################################################################## + +class MetadataTable: + """A complete CCPP metadata table (one ``[ccpp-table-properties]`` block + and all its ``[ccpp-arg-table]`` sections). + + Parameters + ---------- + table_name : str + The ``name`` from ``[ccpp-table-properties]``. + table_type : str + One of ``scheme``, ``host``, ``control``, ``suite``, ``ddt``. + file_path : str + Source file path (used in error messages and ``USE`` statements). + context : ParseContext + The location of the ``[ccpp-table-properties]`` header. + + Examples + -------- + >>> from metadata.parse_tools import ParseContext + >>> ctx = ParseContext(0, 'host.meta') + >>> tbl = MetadataTable('my_module', 'host', 'host.meta', ctx) + >>> tbl.table_name + 'my_module' + >>> tbl.table_type + 'host' + >>> tbl.is_scheme + False + """ + + def __init__(self, table_name: str, table_type: str, + file_path: str, context: ParseContext): + if table_type not in VALID_TABLE_TYPES: + raise CCPPError( + "Table type '{}' is not valid; must be one of {}, at {}".format( + table_type, sorted(VALID_TABLE_TYPES), context + ) + ) + self.table_name: str = table_name + self.table_type: str = table_type + self.file_path: str = file_path + self.context: ParseContext = context + self._sections: List[MetadataSection] = [] + # Optional table-level properties (set by apply_table_props). + self.dependencies: List[str] = [] + self.source_path: str = '' + # Fortran module name that exports this table's Fortran symbols. + # When absent in metadata, falls back to :attr:`table_name` (the + # common case: the .meta file shares its base name with the + # Fortran module). Applies to any table type whose contents are + # imported from a Fortran module (``scheme``, ``host``, ``ddt``); + # the cap generator uses it to emit ``use , only: ...`` + # lines targeting the actual module rather than the table name. + self.module_name: str = '' + # Each entry is ``(kind_name, module, spec)``; aggregated by + # ccpp_capgen into the kind map for ccpp_kinds.F90. + self.kind_specs: List[Tuple[str, str, str]] = [] + + def apply_table_props(self, props: dict) -> None: + """Apply extra ``[ccpp-table-properties]`` key-value pairs to this table. + + Recognised keys + --------------- + ``source_path`` + Relative path from the ``.meta`` file directory to the directory + containing the corresponding Fortran source (``.F90``). Resolved + to an absolute path and stored in :attr:`source_path`. Defaults + to the ``.meta`` file's own directory. + + ``dependencies`` + Comma-separated list of dependency file paths (may include + ``../../``-style relative paths). Resolved against the ``.meta`` + directory, optionally adjusted by ``dependencies_path``. + + ``dependencies_path`` + Relative path from the ``.meta`` directory used as the base for + resolving entries in ``dependencies``. + + ``kind_spec`` + Either a single ``:=>spec`` (or shorthand + ``:``) string, or a list of such strings when the + ``kind_spec`` key appears more than once in the table header. + Each entry is parsed into a ``(kind_name, module, spec)`` triple + and appended to :attr:`kind_specs`. + + Examples + -------- + >>> from metadata.parse_tools import ParseContext + >>> ctx = ParseContext(0, 's.meta') + >>> t = MetadataTable('s', 'scheme', '/project/src/s.meta', ctx) + >>> t.apply_table_props({'source_path': 'fortran', 'dependencies': 'util.F90', 'dependencies_path': 'lib'}) + >>> t.source_path == '/project/src/fortran' + True + >>> t.dependencies + ['/project/src/lib/util.F90'] + >>> t = MetadataTable('s', 'scheme', '/p/s.meta', ctx) + >>> t.apply_table_props({'kind_spec': 'temp_kinds:kind_temp=>temp_r8'}) + >>> t.kind_specs + [('kind_temp', 'temp_kinds', 'temp_r8')] + >>> t = MetadataTable('s', 'scheme', '/p/s.meta', ctx) + >>> t.apply_table_props({'kind_spec': [ + ... 'temp_kinds:kind_temp=>temp_r8', + ... 'host_kinds:kind_r4', + ... ]}) + >>> t.kind_specs + [('kind_temp', 'temp_kinds', 'temp_r8'), ('kind_r4', 'host_kinds', 'kind_r4')] + """ + meta_dir = os.path.dirname(os.path.abspath(self.file_path)) + + if 'source_path' in props: + self.source_path = os.path.normpath( + os.path.join(meta_dir, props['source_path']) + ) + else: + self.source_path = meta_dir + + dep_base = meta_dir + if 'dependencies_path' in props: + dep_base = os.path.normpath( + os.path.join(meta_dir, props['dependencies_path']) + ) + + if 'dependencies' in props: + # ``dependencies`` may legitimately appear more than once in a + # single ``[ccpp-table-properties]`` block; the parser + # collects repeats into a list (analogous to ``kind_spec``). + # A single occurrence still arrives as a string. Each entry + # is a comma-separated list of file paths (or "none" to + # signal an empty dependency set). + raw_entries = props['dependencies'] + if isinstance(raw_entries, str): + raw_entries = [raw_entries] + for raw in raw_entries: + raw = raw.strip() + if raw.lower() == 'none': + continue + for entry in raw.split(','): + entry = entry.strip() + if entry: + self.dependencies.append( + os.path.normpath(os.path.join(dep_base, entry)) + ) + + if 'kind_spec' in props: + raw_specs = props['kind_spec'] + if isinstance(raw_specs, str): + raw_specs = [raw_specs] + for entry in raw_specs: + self.kind_specs.append( + _parse_kind_spec_value(entry, self.context) + ) + + if 'module_name' in props: + raw = props['module_name'].strip() + if raw: + self.module_name = raw + + @property + def is_scheme(self) -> bool: + """True if this is a ``scheme`` table.""" + return self.table_type == SCHEME_TABLE_TYPE + + def sections(self) -> List[MetadataSection]: + """Return all sections (``[ccpp-arg-table]`` blocks) in this table.""" + return list(self._sections) + + def variables(self) -> List[MetaVar]: + """Return all variables across all sections (de-duplicated by standard name). + + For a singleton table (``host``, ``control``, ``ddt``) there is only + one section, so this simply returns that section's variables. For + scheme tables all variables from all phases are returned; variables + that appear in multiple phases are included only once (first occurrence + wins). + """ + seen: Dict[str, MetaVar] = {} + for sec in self._sections: + for var in sec.variables: + if var.standard_name not in seen: + seen[var.standard_name] = var + return list(seen.values()) + + def add_section(self, section: MetadataSection) -> None: + """Append *section* to this table. + + Enforces that singleton table types (``host``, ``control``, ``suite``, + ``ddt``) have at most one section. Also verifies that the section's + type matches the table type. + """ + if section.section_type != self.table_type: + raise CCPPError( + "Section type '{}' does not match table type '{}' " + "in table '{}', at {}".format( + section.section_type, self.table_type, + self.table_name, section.context + ) + ) + if self.table_type in SINGLETON_TABLE_TYPES and self._sections: + raise CCPPError( + "Table type '{}' allows only one section per table; " + "found a second section '{}' in table '{}', at {}".format( + self.table_type, section.section_name, + self.table_name, section.context + ) + ) + self._sections.append(section) + + def section_for_phase(self, phase: str) -> Optional[MetadataSection]: + """Return the section for the given scheme *phase*, or ``None``. + + Only meaningful for scheme tables; always returns ``None`` for + other table types. + """ + for sec in self._sections: + if sec.phase == phase: + return sec + return None + + def __repr__(self) -> str: + return "MetadataTable({!r}, type={!r}, nsections={})".format( + self.table_name, self.table_type, len(self._sections) + ) + + +######################################################################## +# File parser +######################################################################## + +def parse_metadata_file(file_path: str) -> List[MetadataTable]: + """Parse a ``.meta`` file and return all :class:`MetadataTable` objects. + + Parameters + ---------- + file_path : str + Absolute or relative path to the ``.meta`` file. + + Returns + ------- + list of MetadataTable + One entry per ``[ccpp-table-properties]`` block found in the file. + + Raises + ------ + CCPPError + On any structural or content error. The error message includes the + file path and line number. + + Notes + ----- + ``type = module`` is rejected with a descriptive error directing the user + to use ``type = host`` instead (breaking rename from the old generator). + + All blank lines and lines starting with ``#`` or ``;`` are ignored. + + Examples + -------- + >>> import tempfile, os + >>> content = ''' + ... [ccpp-table-properties] + ... name = test_host + ... type = host + ... + ... [ccpp-arg-table] + ... name = test_host + ... type = host + ... [ im ] + ... standard_name = horizontal_dimension + ... units = count + ... dimensions = () + ... type = integer + ... ''' + >>> with tempfile.NamedTemporaryFile(mode='w', suffix='.meta', + ... delete=False) as f: + ... _ = f.write(content) + ... fname = f.name + >>> tables = parse_metadata_file(fname) + >>> os.unlink(fname) + >>> len(tables) + 1 + >>> tables[0].table_name + 'test_host' + >>> tables[0].table_type + 'host' + >>> tables[0].sections()[0].variables[0].standard_name + 'horizontal_dimension' + """ + if not os.path.isfile(file_path): + raise CCPPError("Metadata file '{}' does not exist".format(file_path)) + + with open(file_path, 'r', encoding='utf-8') as fh: + lines = fh.readlines() + + return _parse_lines(lines, file_path) + + +def _parse_lines(lines: List[str], file_path: str) -> List[MetadataTable]: + """Internal line-by-line parser. Exposed for unit-testing without + needing real files. + + Parameters + ---------- + lines : list of str + Source lines (with or without trailing newlines). + file_path : str + Used only for :class:`ParseContext` error messages. + + Returns + ------- + list of MetadataTable + """ + tables: List[MetadataTable] = [] + current_table: Optional[MetadataTable] = None + current_section: Optional[MetadataSection] = None + current_var: Optional[MetaVar] = None + collecting_table_props = False + collecting_section_props = False + + # Accumulate key=value pairs for the current table/section/variable header. + pending_props: Dict[str, str] = {} + pending_start: int = 0 + + def ctx(lineno: int) -> ParseContext: + return ParseContext(linenum=lineno, filename=file_path) + + def flush_var(lineno: int) -> None: + """Validate and attach the buffered variable to the current section.""" + nonlocal current_var + if current_var is None: + return + require_intent = (current_section is not None and + current_section.section_type == SCHEME_TABLE_TYPE) + current_var.validate(require_intent=require_intent, context=ctx(lineno)) + if current_section is None: + raise CCPPError( + "Variable '{}' found outside any section, at {}".format( + current_var.local_name, ctx(lineno) + ) + ) + current_section.add_variable(current_var) + current_var = None + + def flush_section(lineno: int) -> None: + """Attach the current section to the current table.""" + nonlocal current_section + if current_section is None: + return + flush_var(lineno) + if current_table is None: + raise CCPPError( + "Section found outside any table, at {}".format(ctx(lineno)) + ) + current_table.add_section(current_section) + current_section = None + + def flush_table_props() -> None: + """Apply any extra table-property keys to the current table.""" + if current_table is not None and collecting_table_props: + current_table.apply_table_props(pending_props) + + for lineno, raw_line in enumerate(lines): + line = raw_line.rstrip('\n').rstrip('\r') + # Discard any inline ``# ...`` comment so headers, key=value lines, + # and the blank-line check all see the same content the user + # intended as data. + line = _strip_inline_comment(line) + + # ---- [ccpp-table-properties] ---------------------------------------- + if line.strip().lower() == _TABLE_PROPS_HDR: + # Finish whatever we were doing. + flush_table_props() + flush_section(lineno) + if current_table is not None: + tables.append(current_table) + current_table = None + pending_props = {} + pending_start = lineno + collecting_table_props = True + collecting_section_props = False + continue + + # ---- [ccpp-arg-table] ----------------------------------------------- + if line.strip().lower() == _ARG_TABLE_HDR: + flush_table_props() + flush_section(lineno) + collecting_section_props = True + collecting_table_props = False + pending_props = {} + pending_start = lineno + continue + + # ---- Blank / comment lines ------------------------------------------ + if _is_blank(line): + continue + + # ---- Variable header [ name ] ---------------------------------------- + var_match = _VAR_HEADER_RE.match(line) + if var_match: + collecting_table_props = False + collecting_section_props = False + # Flush the previous variable. + flush_var(lineno) + # Start a new variable. + local_name = var_match.group(1) + current_var = MetaVar(local_name, ctx(lineno)) + continue + + # ---- Key = value line ----------------------------------------------- + if collecting_table_props: + pairs = _parse_config_line(line, ctx(lineno)) + for key, val in pairs: + if key in ('kind_spec', 'dependencies'): + # These keys may legitimately appear more than once + # in a single table header; accumulate occurrences + # into a list. ``apply_table_props`` accepts either + # form (string when seen once, list when repeated). + pending_props.setdefault(key, []).append(val) + continue + if key in pending_props: + raise CCPPError( + "Duplicate table property '{}', at {}".format( + key, ctx(lineno) + ) + ) + pending_props[key] = val + # Try to build the MetadataTable as soon as we have name + type. + if 'name' in pending_props and 'type' in pending_props and current_table is None: + current_table = _make_table( + pending_props['name'], pending_props['type'], + file_path, ctx(pending_start) + ) + continue + + if collecting_section_props: + pairs = _parse_config_line(line, ctx(lineno)) + for key, val in pairs: + if key in pending_props: + raise CCPPError( + "Duplicate section property '{}', at {}".format( + key, ctx(lineno) + ) + ) + pending_props[key] = val + # Try to build the MetadataSection as soon as we have name + type. + if ('name' in pending_props and 'type' in pending_props + and current_section is None and current_table is not None): + current_section = _make_section( + pending_props['name'], pending_props['type'], + current_table.table_name, ctx(pending_start) + ) + continue + + if current_var is not None: + pairs = _parse_config_line(line, ctx(lineno)) + for key, val in pairs: + if current_section is not None: + sec_type = current_section.section_type + # ``active`` is a host-model attribute: it expresses a + # condition (referencing host standard names) under + # which a host-owned variable is valid storage. It + # belongs only on host and ddt tables. control vars + # are unconditionally framework-injected, suite tables + # are generated wholesale, and scheme args are never + # the originating storage — so all three reject it. + if (sec_type not in ('host', 'ddt') + and key == 'active'): + raise ParseSyntaxError( + "'active' is a host-model attribute and may " + "only appear in host or ddt metadata; not " + "valid for {} tables".format(sec_type), + token=key, context=ctx(lineno) + ) + if (sec_type != SCHEME_TABLE_TYPE + and key in ('intent', 'optional')): + raise ParseSyntaxError( + "'{}' is a scheme-only attribute and cannot " + "appear in host, control, ddt, or suite " + "metadata".format(key), + token=key, context=ctx(lineno) + ) + if (sec_type != SCHEME_TABLE_TYPE and + key in ('constituent', 'advected', 'molar_mass')): + raise ParseSyntaxError( + "'{}' is a scheme-only constituent hint and cannot " + "appear in host, control, ddt, or suite metadata".format(key), + token=key, context=ctx(lineno) + ) + # auto-clone-constituents: the four legacy + # instantiate-kwarg attrs are scheme-only when the + # shim is enabled. When the shim is off they get + # rejected one layer down (unknown attribute), so + # the guard only matters in shim-on builds. + if (sec_type != SCHEME_TABLE_TYPE + and auto_clone_constituents.is_enabled() + and key in + auto_clone_constituents.extra_known_attrs()): + raise ParseSyntaxError( + "'{}' is a scheme-only constituent property " + "and cannot appear in host, control, ddt, or " + "suite metadata".format(key), + token=key, context=ctx(lineno) + ) + current_var.set_attr(key, val, ctx(lineno)) + continue + + # Line is not blank and not handled — syntax error. + raise ParseSyntaxError("unexpected line", token=line.strip(), + context=ctx(lineno)) + + # ---- End of file: flush any in-progress objects ------------------------- + flush_table_props() + flush_section(len(lines)) + if current_table is not None: + tables.append(current_table) + + return tables + + +def _make_table(name: str, type_str: str, file_path: str, + context: ParseContext) -> MetadataTable: + """Construct a :class:`MetadataTable`, with helpful error for ``type=module``.""" + ttype = type_str.strip().lower() + if ttype == 'module': + raise CCPPError( + "Table type 'module' is not supported; use 'type = host' instead, " + "at {}".format(context) + ) + return MetadataTable(name.strip(), ttype, file_path, context) + + +def _make_section(name: str, type_str: str, table_name: str, + context: ParseContext) -> MetadataSection: + """Construct a :class:`MetadataSection`.""" + return MetadataSection( + name.strip(), type_str.strip().lower(), table_name, context + ) diff --git a/capgen/metadata/parse_tools/__init__.py b/capgen/metadata/parse_tools/__init__.py new file mode 100644 index 00000000..2bb38592 --- /dev/null +++ b/capgen/metadata/parse_tools/__init__.py @@ -0,0 +1,41 @@ +"""Parse utilities shared across metadata parsing and validation.""" + +from .parse_source import ( + CCPPError, + ParseSyntaxError, + ParseInternalError, + ParseContext, + context_string, +) +from .parse_log import init_log, set_log_level, set_log_to_null, set_log_to_stdout +from .parse_checkers import ( + check_units, + check_dimensions, + check_cf_standard_name, + check_diagnostic_fixed, + check_diagnostic_id, + check_fortran_id, + check_fortran_ref, + check_fortran_type, + check_fortran_intrinsic, + check_molar_mass, + # auto-clone-constituents: legacy-shim checkers exported here so + # ``MetaVar.set_attr`` can reach them; gated by the shim's flag at + # the call site, not the import. + check_default_value, + check_min_value, + check_water_species, + check_mixing_ratio_type, + FORTRAN_SCALAR_REF_RE, +) +from .fortran_conditional import FORTRAN_CONDITIONAL_REGEX +from .xml_tools import ( + read_xml_file, + find_schema_version, + expand_nested_suites, + write_xml_file, +) +from .io_helpers import ( + write_if_changed, + open_if_changed, +) diff --git a/scripts/parse_tools/fortran_conditional.py b/capgen/metadata/parse_tools/fortran_conditional.py similarity index 100% rename from scripts/parse_tools/fortran_conditional.py rename to capgen/metadata/parse_tools/fortran_conditional.py diff --git a/capgen/metadata/parse_tools/io_helpers.py b/capgen/metadata/parse_tools/io_helpers.py new file mode 100644 index 00000000..44108fee --- /dev/null +++ b/capgen/metadata/parse_tools/io_helpers.py @@ -0,0 +1,152 @@ +"""File-write helpers with no-op-if-unchanged semantics. + +The original ``ccpp-prebuild`` and ``ccpp-capgen`` both avoided rewriting +generated cap files when their content was unchanged — preserving each +file's mtime so downstream build systems (CMake, Make, Ninja) don't +trigger unnecessary recompilation cascades. This module reproduces that +behaviour for ``capgen``. + +Staging strategy +---------------- +A naive ``open(path, 'w')`` always touches the mtime, even when the +content is identical. Instead each writer builds the file's content in +memory and calls :func:`write_if_changed`, which: + +1. Reads the existing file at *file_path* (if any). +2. If the existing content matches the new content byte-for-byte, returns + ``False`` without touching the filesystem. +3. Otherwise writes the new content to a sibling temp file (in the same + directory, which sits **under the generator's output root** — never + ``/tmp``, so this works on systems that disallow ``/tmp`` writes) and + then ``os.replace``s it over the target. Same-directory replace is + atomic on POSIX and Windows; no partial writes. + +For writers that already produce content via a ``with open(...) as fh`` +pattern, use :func:`open_if_changed` as a drop-in replacement — it yields +a string buffer that gets staged on context exit. +""" + +import io +import logging +import os +import tempfile +from contextlib import contextmanager +from typing import Iterator, Optional + + +def write_if_changed( + file_path: str, + content: str, + encoding: str = 'utf-8', + logger: Optional[logging.Logger] = None, +) -> bool: + """Write *content* to *file_path* iff it differs from existing content. + + Parameters + ---------- + file_path : str + Absolute or relative path of the target file. + content : str + Full file content to write. + encoding : str + Encoding passed to :func:`open` for both the read-back comparison + and the staged write. Defaults to ``'utf-8'`` to match every + capgen writer. + logger : logging.Logger, optional + When supplied, the helper logs an ``info``-level message after + each call: ``"Wrote "`` if the file was newly written or + rewritten, or ``"Unchanged: "`` if the existing content + matched and the filesystem was left untouched. Callers want + this so end users can tell at a glance which generated files + actually changed on a rerun (the original ccpp-prebuild / + ccpp-capgen output distinguished the two cases too). + + Returns + ------- + bool + ``True`` if the file was written or replaced; ``False`` if the + existing content already matched. + + Notes + ----- + The temp file is created in the target's parent directory via + :func:`tempfile.mkstemp` (which generates a unique name and opens it + with ``O_EXCL`` semantics). On any exception, the temp file is + removed so we never leak ``.capgen_tmp_*`` artifacts. Crucially the + staging directory is the target's parent — which is under the + generator's output root — so no ``/tmp`` access is required. + """ + parent = os.path.dirname(os.path.abspath(file_path)) or '.' + os.makedirs(parent, exist_ok=True) + + if os.path.isfile(file_path): + try: + with open(file_path, 'r', encoding=encoding) as fh: + existing = fh.read() + if existing == content: + if logger is not None: + logger.info("Unchanged: %s", file_path) + return False + except (OSError, UnicodeDecodeError): + # Fall through to overwrite if the existing file is + # unreadable or has a different encoding. + pass + + tmp_fd, tmp_path = tempfile.mkstemp( + dir=parent, + prefix='.capgen_tmp_', + suffix='_' + os.path.basename(file_path), + ) + try: + with os.fdopen(tmp_fd, 'w', encoding=encoding) as fh: + fh.write(content) + os.replace(tmp_path, file_path) + except BaseException: + try: + os.unlink(tmp_path) + except OSError: + pass + raise + if logger is not None: + logger.info("Wrote %s", file_path) + return True + + +@contextmanager +def open_if_changed( + file_path: str, + mode: str = 'w', + encoding: str = 'utf-8', + logger: Optional[logging.Logger] = None, +) -> Iterator[io.StringIO]: + """Drop-in replacement for ``open(file_path, 'w', encoding=...)``. + + Yields an in-memory :class:`io.StringIO` buffer. When the context + exits without an exception, the buffer's contents are handed to + :func:`write_if_changed` — so the on-disk file is only touched when + the content actually changes. + + Only text-mode writing is supported; *mode* must be ``'w'`` (the + parameter exists for call-site parity with the stdlib ``open``). + + Example + ------- + Refactor:: + + with open(out_path, 'w', encoding='utf-8') as fh: + fh.write('\\n'.join(lines) + '\\n') + + into:: + + with open_if_changed(out_path) as fh: + fh.write('\\n'.join(lines) + '\\n') + """ + if mode != 'w': + raise ValueError( + "open_if_changed only supports text-write mode 'w'; " + "got mode={!r}".format(mode) + ) + buf = io.StringIO() + yield buf + write_if_changed(file_path, buf.getvalue(), encoding=encoding, + logger=logger) diff --git a/capgen/metadata/parse_tools/parse_checkers.py b/capgen/metadata/parse_tools/parse_checkers.py new file mode 100644 index 00000000..63bed4ab --- /dev/null +++ b/capgen/metadata/parse_tools/parse_checkers.py @@ -0,0 +1,662 @@ +#!/usr/bin/env python3 + +"""Helper functions to validate parsed input.""" + +import re + +from .parse_source import CCPPError + +_UNITLESS_REGEX = "1" +_NON_LEADING_ZERO_NUM = r"[1-9]\d*" +_CHAR_WITH_UNDERSCORE = "([a-zA-Z]+_[a-zA-Z]+)+" +_NEGATIVE_NON_LEADING_ZERO_NUM = f"[-]{_NON_LEADING_ZERO_NUM}" +_POSITIVE_NON_LEADING_ZERO_NUM = f"[+]{_NON_LEADING_ZERO_NUM}" +_UNIT_EXPONENT = f"({_NEGATIVE_NON_LEADING_ZERO_NUM}|{_POSITIVE_NON_LEADING_ZERO_NUM}|{_NON_LEADING_ZERO_NUM})" +_UNIT_REGEX = f"[a-zA-Z]+{_UNIT_EXPONENT}?" +_UNITS_REGEX = rf"^({_CHAR_WITH_UNDERSCORE}|{_UNIT_REGEX}(\s{_UNIT_REGEX})*|{_UNITLESS_REGEX})$" +_UNITS_RE = re.compile(_UNITS_REGEX) +_MAX_MOLAR_MASS = 10000.0 + + +def check_units(test_val, prop_dict, error): + """Return if a valid unit, otherwise, None + if is True, raise an Exception if is not valid. + >>> check_units('m s-1', None, True) + 'm s-1' + >>> check_units('kg m-3', None, True) + 'kg m-3' + >>> check_units('m2 s-2', None, True) + 'm2 s-2' + >>> check_units('m+2 s-2', None, True) + 'm+2 s-2' + >>> check_units('1', None, True) + '1' + >>> check_units('', None, False) + + >>> check_units('', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: '' is not a valid unit + >>> check_units(['foo'], None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: ['foo'] is invalid; not a string + """ + if isinstance(test_val, str): + if _UNITS_RE.match(test_val.strip()) is None: + if error: + raise CCPPError("'{}' is not a valid unit".format(test_val)) + test_val = None + else: + if error: + raise CCPPError("'{}' is invalid; not a string".format(test_val)) + test_val = None + return test_val + + +def check_dimensions(test_val, prop_dict, error, max_len=0): + """Return if a valid dimensions list, otherwise, None + If > 0, each string in must not be longer than + . + if is True, raise an Exception if is not valid. + >>> check_dimensions(["dim1", "dim2name"], None, False) + ['dim1', 'dim2name'] + >>> check_dimensions([":", ":"], None, False) + [':', ':'] + >>> check_dimensions(["8", "::"], None, False) + ['8', '::'] + >>> check_dimensions(['start1:end1', 'start2:end2'], None, False) + ['start1:end1', 'start2:end2'] + >>> check_dimensions(['size(foo)'], None, False) + ['size(foo)'] + >>> check_dimensions(['size(foo,1'], None, False) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: Invalid dimension component, size(foo,1 + >>> check_dimensions(["dim1", "dim2name"], None, True, max_len=5) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'dim2name' is too long (> 5 chars) + >>> check_dimensions("hi_mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'hi_mom' is invalid; not a list + >>> check_dimensions(["ccpp_constant_one:1", "dim2name"], None, True) + ['ccpp_constant_one:1', 'dim2name'] + """ + if not isinstance(test_val, list): + if error: + raise CCPPError("'{}' is invalid; not a list".format(test_val)) + return None + for item in test_val: + isplit = item.split(':') + if len(isplit) > 3: + if error: + raise CCPPError("'{}' is an invalid dimension range".format(item)) + return None + # Integer literals are valid in any bound position; semantic + # restrictions (e.g. horizontal_dimension lower bound must be 1) + # are enforced by the resolver, not here. + tdims = [x.strip() for x in isplit if len(x) > 0] + for tdim in tdims: + try: + int(tdim) + valid = True + except ValueError: + valid = check_fortran_id(tdim, None, error, + max_len=max_len) is not None + if not valid and tdim.strip().lower()[0:4] == 'size': + if -1 in check_balanced_paren(tdim[4:]): + raise CCPPError( + 'Invalid dimension component, {}'.format(tdim)) + valid = True + if not valid: + if error: + raise CCPPError(f"'{item}' is an invalid dimension name") + return None + return test_val + + +CF_ID = r"(?i)[a-z][a-z0-9_]*" +__CFID_RE = re.compile(CF_ID + r"$") + + +def check_cf_standard_name(test_val, prop_dict, error): + """Return if a valid CF Standard Name, otherwise, None. + http://cfconventions.org/Data/cf-standard-names/docs/guidelines.html + if is True, raise an Exception if is not valid. + >>> check_cf_standard_name("hi_mom", None, False) + 'hi_mom' + >>> check_cf_standard_name("hi mom", None, False) + + >>> check_cf_standard_name("", None, False) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: CCPP Standard Name cannot be blank + >>> check_cf_standard_name("Agood4tranID", None, False) + 'agood4tranid' + """ + if len(test_val) == 0: + raise CCPPError("CCPP Standard Name cannot be blank") + if __CFID_RE.match(test_val) is None: + if error: + raise CCPPError( + "'{}' is not a valid CCPP Standard Name".format(test_val)) + return None + return test_val.lower() + + +FORTRAN_ID = r"([A-Za-z][A-Za-z0-9_]*)" +__FID_RE = re.compile(FORTRAN_ID + r"$") +# Scalar array-reference pattern below allows `:` placeholders, unlike +# real Fortran code, so the same regex covers both scalar refs and +# slice descriptors in metadata. +__FORTRAN_AID = r"(?:[A-Za-z][A-Za-z0-9_]*)" +__FORT_INT = r"[0-9]+" +__FORT_DIM = r"(?:" + __FORTRAN_AID + r"|[:]|" + __FORT_INT + r")" +__REPEAT_DIM = r"(?:,\s*" + __FORT_DIM + r"\s*)" +__FORTRAN_SCALAR_ARREF = r"[(]\s*(" + __FORT_DIM + r"\s*" + __REPEAT_DIM + r"{0,6})[)]" +FORTRAN_SCALAR_REF_RE = re.compile( + r"(?:" + FORTRAN_ID + r"\s*" + __FORTRAN_SCALAR_ARREF + r")$") +FORTRAN_INTRINSIC_TYPES = ["integer", "real", "logical", "complex", + "double precision", "character"] +FORTRAN_DP_RE = re.compile(r"(?i)double\s*precision") + +_REGISTERED_FORTRAN_DDT_NAMES = ["ccpp_constituent_prop_ptr_t"] + + +def check_fortran_id(test_val, prop_dict, error, max_len=0): + """Return if a valid Fortran identifier, otherwise, None + If > 0, must not be longer than . + if is True, raise an Exception if is not valid. + >>> check_fortran_id("hi_mom", None, False) + 'hi_mom' + >>> check_fortran_id("hi_mom", None, True, max_len=5) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'hi_mom' is too long (> 5 chars) + >>> check_fortran_id("hi mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'hi_mom' is not a valid Fortran identifier + >>> check_fortran_id("2pac", None, False) + + >>> check_fortran_id("Agood4tranID", None, False) + 'Agood4tranID' + """ + if __FID_RE.match(test_val) is None: + if error: + raise CCPPError( + "'{}' is not a valid Fortran identifier".format(test_val)) + return None + if max_len > 0 and len(test_val) > max_len: + if error: + raise CCPPError( + "'{}' is too long (> {} chars)".format(test_val, max_len)) + return None + return test_val + + +def check_fortran_ref(test_val, prop_dict, error, max_len=0): + """Return if a valid simple Fortran variable reference, + otherwise, None. A simple Fortran variable reference is defined as + a scalar id or a scalar array reference. + if is True, raise an Exception if is not valid. + >>> check_fortran_ref("hi_mom", None, False) + 'hi_mom' + >>> check_fortran_ref("hi mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'hi_mom' is not a valid Fortran identifier + >>> check_fortran_ref("foo(bar)", None, False) + 'foo(bar)' + >>> check_fortran_ref("foo( bar, baz )", None, False) + 'foo( bar, baz )' + >>> check_fortran_ref("foo( :, baz )", None, False) + 'foo( :, baz )' + >>> check_fortran_ref("foo( bar, )", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'foo( bar, )' is not a valid Fortran scalar reference + >>> check_fortran_ref("foo(bar, bazz)", None, True, max_len=3) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'bazz' is too long (> 3 chars) in foo(bar, bazz) + """ + idval = check_fortran_id(test_val, prop_dict, False, max_len=max_len) + if idval is not None: + return test_val + if FORTRAN_SCALAR_REF_RE.match(test_val) is None: + if error: + raise CCPPError( + "'{}' is not a valid Fortran scalar reference".format(test_val)) + return None + if max_len > 0: + tokens = test_val.strip().rstrip(')').split('(') + tokens = [tokens[0].strip()] + [x.strip() for x in tokens[1].split(',')] + for token in tokens: + if len(token) > max_len: + if error: + raise CCPPError( + "'{}' is too long (> {} chars) in {}".format( + token, max_len, test_val)) + return None + return test_val + + +def check_fortran_intrinsic(typestr, error=False): + """Return if a valid Fortran intrinsic type, otherwise, None + if is True, raise an Exception if is not valid. + >>> check_fortran_intrinsic("real", error=False) + 'real' + >>> check_fortran_intrinsic("InteGer") + 'InteGer' + >>> check_fortran_intrinsic("double precision") + 'double precision' + >>> check_fortran_intrinsic("doubleprecision") + 'doubleprecision' + >>> check_fortran_intrinsic("char", error=True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'char' is not a valid Fortran type + >>> check_fortran_intrinsic("complex(kind=r8)") + + """ + chk_type = typestr.strip().lower() + match = chk_type in FORTRAN_INTRINSIC_TYPES + if not match and chk_type[0:6] == 'double': + match = FORTRAN_DP_RE.match(chk_type) is not None + if not match: + if error: + raise CCPPError("'{}' is not a valid Fortran type".format(typestr)) + return None + return typestr + + +def check_fortran_type(typestr, prop_dict, error): + """Return if a valid Fortran type, otherwise, None + if is True, raise an Exception if is not valid. + >>> check_fortran_type("real", None, False) + 'real' + >>> check_fortran_type("char", {}, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'char' is not a valid Fortran type + >>> check_fortran_type("type", {}, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'type' is not a valid derived Fortran type + """ + dt = "" + match = check_fortran_intrinsic(typestr, error=False) + if match is None: + match = registered_fortran_ddt_name(typestr) + dt = " derived" + if match is None: + if error: + raise CCPPError( + "'{}' is not a valid{} Fortran type".format(typestr, dt)) + return None + return typestr + + +def check_diagnostic_fixed(test_val, prop_dict, error): + """Return if a valid descriptor for a CCPP diagnostic, + otherwise, None. + If is True, raise an Exception if is not valid. + A fixed diagnostic name is any Fortran identifier, however, it is + an error to specify both 'diagnostic_name' and 'diagnostic_name_fixed'. + >>> check_diagnostic_fixed("foo", {'diagnostic_name_fixed' : 'foo'}, False) + 'foo' + >>> check_diagnostic_fixed("foo", {'diagnostic_name':'foo','local_name':'hi','standard_name':'mom'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: hi (mom) cannot have both 'diagnostic_name' and 'diagnostic_name_fixed' attributes + >>> check_diagnostic_fixed("2foo", {'diagnostic_name_fixed':'foo','local_name':'hi','standard_name':'mom'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: '2foo' (hi) is not a valid fixed diagnostic name + """ + if (prop_dict and ('diagnostic_name' in prop_dict) and + prop_dict['diagnostic_name']): + if error: + lname = prop_dict.get('local_name', 'UNKNOWN') + sname = prop_dict.get('standard_name', 'UNKNOWN') + raise CCPPError( + "{} ({}) cannot have both 'diagnostic_name' and " + "'diagnostic_name_fixed' attributes".format(lname, sname)) + return None + if check_fortran_id(test_val, prop_dict, False) is None: + if error: + lname = prop_dict.get('local_name', 'UNKNOWN') + raise CCPPError( + "'{}' ({}) is not a valid fixed diagnostic name".format( + test_val, lname)) + return None + return test_val + + +_DIAG_PRE = r"(" + FORTRAN_ID + ")?" +_DIAG_SUFF = r"([_0-9A-Za-z]+)?" +_DIAG_PROP = r"((\${process}|\${scheme_name})" + _DIAG_SUFF + r")" +_DIAG_RE = re.compile(_DIAG_PRE + _DIAG_PROP + r"?$") + + +def check_diagnostic_id(test_val, prop_dict, error): + """Return if a valid descriptor for a CCPP diagnostic, + otherwise, None. + If is True, raise an Exception if is not valid. + A diagnostic name is a Fortran identifier with the optional + addition of one variable substitution. + A variable substitution is a substring of the form of either: + ${process}: The scheme process name will be substituted for this + substring. If this substring is included, it is an error for + there to be no process specified by the scheme (although this + error cannot be detected by this routine). + ${scheme_name}: The scheme name will be substituted for this substring. + It is an error to specify both 'diagnostic_name' and + 'diagnostic_name_fixed'. + >>> check_diagnostic_id("foo", {'diagnostic_name' : 'foo'}, False) + 'foo' + >>> check_diagnostic_id("foo_${process}", {}, False) + 'foo_${process}' + >>> check_diagnostic_id("foo_${scheme_name}_2bad", {}, False) + 'foo_${scheme_name}_2bad' + >>> check_diagnostic_id("pref_${scheme}_suff", {}, False) + + >>> check_diagnostic_id("foo", {'diagnostic_name_fixed':'foo','local_name':'hi','standard_name':'mom'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: hi (mom) cannot have both 'diagnostic_name' and 'diagnostic_name_fixed' attributes + """ + if (prop_dict and ('diagnostic_name_fixed' in prop_dict) and + prop_dict['diagnostic_name_fixed']): + if error: + lname = prop_dict.get('local_name', 'UNKNOWN') + sname = prop_dict.get('standard_name', 'UNKNOWN') + raise CCPPError( + "{} ({}) cannot have both 'diagnostic_name' and " + "'diagnostic_name_fixed' attributes".format(lname, sname)) + return None + if _DIAG_RE.match(test_val) is None: + if error: + raise CCPPError( + "'{}' is not a valid diagnostic_name value".format(test_val)) + return None + return test_val + + +def check_molar_mass(test_val, prop_dict, error): + """Return if valid molar mass, otherwise, None + if is True, raise an Exception if is not valid. + >>> check_molar_mass('1', None, True) + 1.0 + >>> check_molar_mass('1.0', None, False) + 1.0 + >>> check_molar_mass('-1', None, False) + + >>> check_molar_mass('-1', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: '-1' is not a valid molar mass + >>> check_molar_mass(10001, None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: '10001' is not a valid molar mass + """ + try: + test_val = float(test_val) + except (TypeError, ValueError): + if error: + raise CCPPError(f"{test_val} is invalid; not a float or int") + return None + if test_val < 0.0 or test_val > _MAX_MOLAR_MASS: + if error: + raise CCPPError(f"{test_val} is not a valid molar mass") + return None + return test_val + + +# auto-clone-constituents: BEGIN legacy-shim checkers. +# These four checkers validate the metadata attributes that the +# ``--legacy-auto-clone-constituents`` shim accepts on scheme args +# (default_value, min_value, water_species, mixing_ratio_type). +# Delete this whole BEGIN..END block when the shim is removed; the +# rest of the file is unchanged. + +# Whitelist of mixing_ratio_type values accepted by the framework. +# Mirrors the canonical set used by original capgen / ccpp-physics +# hosts. Audit before extending — adding a name silently accepts it +# everywhere. +_MIXING_RATIO_TYPES = frozenset({'dry', 'wet', 'wrt_dry', 'wrt_moist'}) + + +# Anchored Fortran real-literal pattern. Captures the numeric body; +# anything that follows must be either empty or a kind suffix. +# Forms accepted by the body: +# 123 .5 1. 1.5 1.5e-3 1.5E+10 1.5d0 1.5D-12 +_FORTRAN_REAL_BODY_RE = re.compile( + r'^\s*([+-]?(?:\d+\.\d*|\.\d+|\d+)(?:[eEdD][+-]?\d+)?)' +) +# Kind suffix: ``_``. Identifier may itself +# contain underscores (``kind_phys``, ``kind_dyn``). +_FORTRAN_KIND_SUFFIX_RE = re.compile(r'^_[A-Za-z0-9_]+$') + + +def _parse_fortran_real_literal(value): + """Convert a Fortran real-literal string to a Python float. + + Legacy CAM-SIMA metadata writes constituent property values in + Fortran source form, with a kind suffix (``0.0_kind_phys``, + ``1.0e-12_kind_dyn``, ``-3.14_8``) and/or a double-precision + exponent marker (``1.0d-5`` / ``1.0D-5``). Python's :func:`float` + rejects both. This helper isolates the numeric body via an + anchored regex (so identifiers like ``not_a_number`` never match), + verifies any trailing token is a valid kind suffix, rewrites + ``d/D`` exponent markers to ``e/E``, and hands the cleaned body + to :func:`float`. + + Raises :class:`ValueError` (just like :func:`float`) on anything + that doesn't parse, so the calling check_X helper handles the + error path uniformly. + + >>> _parse_fortran_real_literal('0.0') + 0.0 + >>> _parse_fortran_real_literal('0.0_kind_phys') + 0.0 + >>> _parse_fortran_real_literal('1.0e-12_kind_dyn') + 1e-12 + >>> _parse_fortran_real_literal('-3.14_8') + -3.14 + >>> _parse_fortran_real_literal('1.0d-5') + 1e-05 + >>> _parse_fortran_real_literal('1.0D0') + 1.0 + >>> _parse_fortran_real_literal('garbage') #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ValueError: ... + >>> _parse_fortran_real_literal('1.0_bad-kind') #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ValueError: ... + """ + s = str(value).strip() + m = _FORTRAN_REAL_BODY_RE.match(s) + if not m: + raise ValueError( + "{!r} does not start with a Fortran real literal".format(value) + ) + body = m.group(1) + rest = s[m.end():] + if rest and not _FORTRAN_KIND_SUFFIX_RE.match(rest): + raise ValueError( + "{!r} has trailing junk after the numeric body".format(value) + ) + # Rewrite double-precision exponent markers (d/D) -> (e/E). + body = body.replace('d', 'e').replace('D', 'E') + return float(body) + + +def check_default_value(test_val, prop_dict, error): + """Return as a float if a valid default_value, otherwise None. + + Accepts any finite Fortran real literal: no positivity constraint + (some species use a negative sentinel as their "uninitialized" + placeholder, see CAM-SIMA cld_liq.F90). The Fortran kind suffix + (e.g. ``0.0_kind_phys``) is stripped before conversion. + + >>> check_default_value('0.0', None, True) + 0.0 + >>> check_default_value('0.0_kind_phys', None, True) + 0.0 + >>> check_default_value('-1.0e30', None, True) + -1e+30 + >>> check_default_value('1.0d-5_kind_phys', None, True) + 1e-05 + >>> check_default_value('not a number', None, False) + + >>> check_default_value('not a number', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'not a number' is not a valid default_value + """ + try: + return _parse_fortran_real_literal(test_val) + except (TypeError, ValueError): + if error: + raise CCPPError("'{}' is not a valid default_value".format(test_val)) + return None + + +def check_min_value(test_val, prop_dict, error): + """Return as a float if a valid min_value, otherwise None. + + Same shape as :func:`check_default_value`; no constraint beyond + "finite float" since min_value is a host-runtime guardrail and the + sensible range is species-dependent. Fortran kind suffix + accepted. + + >>> check_min_value('0.0', None, True) + 0.0 + >>> check_min_value('1.0e-12', None, True) + 1e-12 + >>> check_min_value('1.0e-12_kind_phys', None, True) + 1e-12 + >>> check_min_value('garbage', None, False) + + >>> check_min_value('garbage', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'garbage' is not a valid min_value + """ + try: + return _parse_fortran_real_literal(test_val) + except (TypeError, ValueError): + if error: + raise CCPPError("'{}' is not a valid min_value".format(test_val)) + return None + + +def check_water_species(test_val, prop_dict, error): + """Return ``True`` / ``False`` for a valid water_species value, + otherwise None. + + Accepts the same surface as the existing ``advected`` / + ``constituent`` flag parsing (``True``/``False``/``T``/``F``, + case-insensitive); rejects anything else. + + >>> check_water_species('True', None, True) + True + >>> check_water_species('false', None, True) + False + >>> check_water_species('maybe', None, False) + + >>> check_water_species('maybe', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'maybe' is not a valid water_species (expected True or False) + """ + if isinstance(test_val, bool): + return test_val + tok = str(test_val).strip().lower() + if tok in ('true', 't', '.true.'): + return True + if tok in ('false', 'f', '.false.'): + return False + if error: + raise CCPPError( + "'{}' is not a valid water_species " + "(expected True or False)".format(test_val) + ) + return None + + +def check_mixing_ratio_type(test_val, prop_dict, error): + """Return if a valid mixing_ratio_type, otherwise None. + + Compared case-insensitively against the canonical whitelist + (``_MIXING_RATIO_TYPES``). Returns the lowercased form on success. + + >>> check_mixing_ratio_type('dry', None, True) + 'dry' + >>> check_mixing_ratio_type('WRT_MOIST', None, True) + 'wrt_moist' + >>> check_mixing_ratio_type('bogus', None, False) + + >>> check_mixing_ratio_type('bogus', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'bogus' is not a valid mixing_ratio_type + """ + if test_val is None: + if error: + raise CCPPError("None is not a valid mixing_ratio_type") + return None + tok = str(test_val).strip().lower() + if tok in _MIXING_RATIO_TYPES: + return tok + if error: + raise CCPPError( + "'{}' is not a valid mixing_ratio_type " + "(expected one of: {})".format( + test_val, ', '.join(sorted(_MIXING_RATIO_TYPES)) + ) + ) + return None + +# auto-clone-constituents: END legacy-shim checkers. + + +def check_balanced_paren(string, start=0, error=False): + """Return indices delineating a balance set of parentheses. + Parentheses in character context do not count. + Left parenthesis search begins at . + Return start and end indices if found + If no parentheses are found, return (-1, -1). + If a left parenthesis is found but no balancing right, return (begin, -1) + where begin is the index where the left parenthesis was found. + If error is True, raise a CCPPError. + >>> check_balanced_paren("foo") + (-1, -1) + >>> check_balanced_paren("(foo, bar)") + (0, 9) + >>> check_balanced_paren("(size(foo,1), qux)") + (0, 17) + >>> check_balanced_paren("(foo('bar()'))") + (0, 13) + >>> check_balanced_paren("(foo('bar()')") + (0, -1) + >>> check_balanced_paren("(foo('bar()')", error=True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: ERROR: Unbalanced parenthesis in '(foo('bar()')' + """ + index = start + begin = -1 + end = -1 + depth = 0 + inchar = None + str_len = len(string) + while index < str_len: + c = string[index] + if c in ('"', "'"): + if inchar == c: + inchar = None + elif inchar is None: + inchar = c + elif inchar is not None: + pass + elif c == '(': + if depth == 0: + begin = index + depth += 1 + elif c == ')': + depth -= 1 + if depth == 0: + end = index + break + index += 1 + if begin >= 0 and end < 0 and error: + raise CCPPError("ERROR: Unbalanced parenthesis in '{}'".format(string)) + return begin, end + + +def registered_fortran_ddt_name(name): + if name in _REGISTERED_FORTRAN_DDT_NAMES: + return name + return None diff --git a/capgen/metadata/parse_tools/parse_log.py b/capgen/metadata/parse_tools/parse_log.py new file mode 100644 index 00000000..fe080cc6 --- /dev/null +++ b/capgen/metadata/parse_tools/parse_log.py @@ -0,0 +1,42 @@ +"""Shared logger utilities for parse processes.""" + +import logging + + +def init_log(name, level=None): + """Initialize and return a named logger. + + Defaults to WARNING level when *level* is not specified and the logger + has no existing level set. + + >>> logger = init_log('test_logger') + >>> logger.name + 'test_logger' + """ + logger = logging.getLogger(name) + llevel = logger.getEffectiveLevel() + if level is None and llevel == logging.NOTSET: + logger.setLevel(logging.WARNING) + elif level: + logger.setLevel(level) + set_log_to_stdout(logger) + return logger + + +def set_log_level(logger, level): + logger.setLevel(level) + + +def _remove_handlers(logger): + for handler in list(logger.handlers): + logger.removeHandler(handler) + + +def set_log_to_stdout(logger): + _remove_handlers(logger) + logger.addHandler(logging.StreamHandler()) + + +def set_log_to_null(logger): + _remove_handlers(logger) + logger.addHandler(logging.NullHandler()) diff --git a/capgen/metadata/parse_tools/parse_source.py b/capgen/metadata/parse_tools/parse_source.py new file mode 100644 index 00000000..6648f59d --- /dev/null +++ b/capgen/metadata/parse_tools/parse_source.py @@ -0,0 +1,118 @@ +"""Parsing primitives: parse context and exception types.""" + +import logging +import os.path + + +def context_string(context=None, with_comma=True, nodir=False): + """Return a human-readable location string from *context*. + + Parameters + ---------- + context : ParseContext or None + Parsing location. ``None`` returns an empty string. + with_comma : bool + Prepend ``', at '`` or ``', in '`` when *context* is given. + nodir : bool + Strip the directory portion of the filename. + + >>> context_string() + '' + >>> context_string(context=ParseContext(linenum=32, filename="dir/source.F90"), with_comma=False) + 'dir/source.F90:33' + >>> context_string(context=ParseContext(linenum=32, filename="dir/source.F90"), with_comma=True) + ', at dir/source.F90:33' + >>> context_string(context=ParseContext(filename="dir/source.F90"), with_comma=False) + 'dir/source.F90' + >>> context_string(context=ParseContext(linenum=32, filename="dir/source.F90"), with_comma=False, nodir=True) + 'source.F90:33' + """ + if context is None: + return '' + if context.line_num < 0: + where_str = 'in ' + else: + where_str = 'at ' + comma = ', ' if with_comma else '' + if not with_comma: + where_str = '' + spec = '{ctx:nodir}' if nodir else '{ctx}' + return ('{comma}{where_str}' + spec).format(comma=comma, where_str=where_str, ctx=context) + + +class CCPPError(ValueError): + """User-facing error with a plain message and no traceback noise.""" + + def __init__(self, message): + logging.shutdown() + super().__init__(message) + + +class ParseSyntaxError(CCPPError): + """Syntax error that includes parsing context in the message.""" + + def __init__(self, token_type, token=None, context=None): + logging.shutdown() + cstr = context_string(context) + if token is None: + message = "{}{}".format(token_type, cstr) + else: + message = "Invalid {}, '{}'{}".format(token_type, token, cstr) + super().__init__(message) + + +class ParseInternalError(Exception): + """Internal parser logic error — not caught by normal user-error handlers.""" + + def __init__(self, errmsg, context=None): + logging.shutdown() + message = "{}{}".format(errmsg, context_string(context)) + super().__init__(message) + + +class ParseContext: + """File-position record used as the location anchor for parse errors. + + Holds a filename and a zero-based line number (negative means «file + level, no specific line»); formats as ``filename:line``. + + >>> str(ParseContext(linenum=0, filename="foo.F90")) + 'foo.F90:1' + >>> str(ParseContext(filename="foo.F90")) + 'foo.F90' + >>> ParseContext(linenum="bad", filename="f.F90") #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + CCPPError: ParseContext linenum must be an int + """ + + def __init__(self, linenum=None, filename=None): + if linenum is None: + linenum = -1 + elif not isinstance(linenum, int): + raise CCPPError('ParseContext linenum must be an int') + + if filename is None: + filename = "" + elif not isinstance(filename, str): + raise CCPPError('ParseContext filename must be a string') + + self.__linenum = linenum + self.__filename = filename + + @property + def line_num(self): + return self.__linenum + + @property + def filename(self): + return self.__filename + + def __format__(self, spec): + fname = os.path.basename(self.__filename) if spec == 'nodir' else self.__filename + if self.__linenum >= 0: + return "{}:{}".format(fname, self.__linenum + 1) + return fname + + def __str__(self): + return format(self) diff --git a/capgen/metadata/parse_tools/xml_tools.py b/capgen/metadata/parse_tools/xml_tools.py new file mode 100644 index 00000000..52f80c12 --- /dev/null +++ b/capgen/metadata/parse_tools/xml_tools.py @@ -0,0 +1,472 @@ +#!/usr/bin/env python3 + +"""XML helpers: schema-version probing, schema validation, nested-suite +expansion, and pretty-printed XML writing. +""" + +import os +import re +import shutil +import subprocess +import sys +import xml.etree.ElementTree as ET +import xml.dom.minidom + +from .parse_source import CCPPError +from .parse_log import init_log, set_log_to_null + +_INDENT_STR = " " +beg_tag_re = re.compile(r"([<][^/][^<>]*[^/][>])") +end_tag_re = re.compile(r"([<][/][^<>/]+[>])") +simple_tag_re = re.compile(r"([<][^/][^<>/]+[/][>])") + +PYSUBVER = sys.version_info[1] +_LOGGER = None + + +class XMLToolsInternalError(ValueError): + """Internal error raised by helpers in this module.""" + + +def find_schema_version(root): + """Return the schema version as ``[major, minor]`` from the *root*'s + ``version`` attribute. + + >>> find_schema_version(ET.fromstring('')) + [1, 0] + >>> find_schema_version(ET.fromstring('')) + [2, 0] + >>> find_schema_version(ET.fromstring('')) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: Illegal version string, '1.a' + Format must be . + >>> find_schema_version(ET.fromstring('')) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: Illegal version string, '0.0' + Major version must be at least 1 + """ + if 'version' not in root.attrib: + raise CCPPError("Version attribute required") + version = root.attrib['version'] + versplit = version.split('.') + try: + if len(versplit) != 2: + raise CCPPError('Major and minor version required') + try: + verbits = [int(x) for x in versplit] + except ValueError as verr: + raise CCPPError(verr) from verr + if verbits[0] < 1: + raise CCPPError('Major version must be at least 1') + if verbits[1] < 0: + raise CCPPError('Minor version must be non-negative') + except CCPPError as verr: + errstr = """Illegal version string, '{}' + Format must be .""" + ve_str = str(verr) + if ve_str: + errstr = ve_str + '\n' + errstr + raise CCPPError(errstr.format(version)) from verr + return verbits + + +def find_schema_file(schema_root, version, schema_path=None): + """Return ``_v_.xsd`` under *schema_path* + (or the current directory), or ``None`` if no such file exists.""" + verstring = '_'.join([str(x) for x in version]) + schema_filename = "{}_v{}.xsd".format(schema_root, verstring) + if schema_path: + schema_file = os.path.join(schema_path, schema_filename) + else: + schema_file = schema_filename + if os.path.exists(schema_file): + return schema_file + return None + + +def validate_xml_file(filename, schema_root, version, logger, schema_path=None): + """Validate *filename* against the matching schema using xmllint.""" + if not os.path.isfile(filename): + raise CCPPError("validate_xml_file: Filename, '{}', does not exist".format(filename)) + if not os.access(filename, os.R_OK): + raise CCPPError("validate_xml_file: Cannot open '{}'".format(filename)) + if os.path.isfile(schema_root): + schema_file = schema_root + else: + if not schema_path: + thispath = os.path.abspath(__file__) + pdir = os.path.dirname(os.path.dirname(os.path.dirname(thispath))) + schema_path = os.path.join(pdir, 'schema') + schema_file = find_schema_file(schema_root, version, schema_path) + if not (schema_file and os.path.isfile(schema_file)): + verstring = '.'.join([str(x) for x in version]) + raise CCPPError( + f"validate_xml_file: Cannot find schema for version {verstring},\n" + f" {schema_file} does not exist" + ) + if not os.access(schema_file, os.R_OK): + raise CCPPError( + "validate_xml_file: Cannot open schema, '{}'".format(schema_file)) + + xmllint = shutil.which('xmllint') + if not xmllint: + raise CCPPError( + "validate_xml_file: xmllint not found, could not validate file {}".format(filename)) + + logger.debug("Checking file {} against schema {}".format(filename, schema_file)) + cmd = [xmllint, '--noout', '--schema', schema_file, filename] + cproc = subprocess.run(cmd, check=False, capture_output=True) + if cproc.returncode == 0: + # Some xmllint builds return 0 even when validation fails; double + # check by looking for the literal 'validates' marker in output. + result = b'validates' in cproc.stdout or b'validates' in cproc.stderr + else: + result = False + if result: + logger.debug(cproc.stdout) + logger.debug(cproc.stderr) + return result + cmd_str = ' '.join(cmd) + outstr = f"Execution of '{cmd_str}' failed with code: {cproc.returncode}\n" + if cproc.stdout: + outstr += f"{cproc.stdout.decode('utf-8', errors='replace').strip()}\n" + if cproc.stderr: + outstr += f"{cproc.stderr.decode('utf-8', errors='replace').strip()}\n" + raise CCPPError(outstr) + + +def read_xml_file(filename, logger=None): + """Read *filename* and return ``(tree, root)``. + + Raises ``CCPPError`` if the file is missing or unreadable. + """ + if os.path.isfile(filename) and os.access(filename, os.R_OK): + with open(filename, 'r', encoding='utf-8') as fh: + try: + tree = ET.parse(fh) + root = tree.getroot() + except ET.ParseError as perr: + raise CCPPError( + "read_xml_file: Cannot read {}, {}".format(filename, perr) + ) from perr + elif not os.access(filename, os.R_OK): + raise CCPPError("read_xml_file: Cannot open '{}'".format(filename)) + else: + raise CCPPError( + "read_xml_file: Filename, '{}', does not exist".format(filename)) + if logger: + logger.debug(f"Reading XML file {filename}") + return tree, root + + +def load_suite_by_name(suite_name, group_name, file, logger=None): + """Load and return a suite or group element from a SDF file. + + Parameters + ---------- + suite_name : str + Name of the suite element to find. + group_name : str or None + Name of the group within the suite to find; ``None`` returns the + whole suite. + file : str + Path to the XML file. + logger : logging.Logger, optional + + Returns + ------- + xml.etree.ElementTree.Element + The matching suite or group element. + + Examples + -------- + >>> import tempfile + >>> import xml.etree.ElementTree as ET + >>> logger = init_log('xml_tools') + >>> set_log_to_null(logger) + >>> tmpdir = tempfile.TemporaryDirectory() + >>> file1_path = os.path.join(tmpdir.name, "file1.xml") + >>> with open(file1_path, "w") as f: + ... _ = f.write(''' + ... + ... + ... + ... + ... ''') + >>> load_suite_by_name("physics_suite", None, file1_path, logger).tag + 'suite' + >>> load_suite_by_name("physics_suite", "dynamics", file1_path, logger).attrib['name'] + 'dynamics' + >>> load_suite_by_name("physics_suite", "missing_group", file1_path, logger) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + CCPPError: Nested suite physics_suite, group missing_group, not found + >>> load_suite_by_name("missing_suite", None, file1_path, logger) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + CCPPError: Nested suite missing_suite not found + >>> tmpdir.cleanup() + """ + _, root = read_xml_file(file, logger) + try: + schema_version = find_schema_version(root) + except CCPPError as verr: + raise CCPPError( + f"{verr} in nested suite XML file '{file}'" + ) from verr + if not validate_xml_file(file, 'suite', schema_version, logger): + raise CCPPError(f"Invalid suite definition file, '{file}'") + if root.attrib.get("name") == suite_name: + if group_name: + for group in root.findall("group"): + if group.attrib.get("name") == group_name: + return group + else: + return root + emsg = f"Nested suite {suite_name}" \ + + (f", group {group_name}," if group_name else "") \ + + " not found" + (f" in file {file}" if file else "") + raise CCPPError(emsg) + + +def replace_nested_suite(element, nested_suite, default_path, logger): + """Replace a ```` element with the suite/group it references. + + Parameters + ---------- + element : Element + Parent of *nested_suite*. + nested_suite : Element + The ```` element to replace. + default_path : str + Directory to resolve a relative ``file=`` attribute against. + logger : logging.Logger or None + + Returns + ------- + str + Name of the suite that was substituted in. + + Examples + -------- + >>> import tempfile + >>> import xml.etree.ElementTree as ET + >>> logger = init_log('xml_tools') + >>> set_log_to_null(logger) + >>> tmpdir = tempfile.TemporaryDirectory() + >>> file1_path = os.path.join(tmpdir.name, "file1.xml") + >>> with open(file1_path, "w") as f: + ... _ = f.write(''' + ... + ... + ... my_scheme + ... + ... + ... ''') + >>> xml = f''' + ... + ... + ... + ... ''' + >>> top_suite = ET.fromstring(xml) + >>> nested = top_suite.find("nested_suite") + >>> replace_nested_suite(top_suite, nested, tmpdir.name, logger) + 'my_suite' + >>> [child.tag for child in top_suite] + ['group'] + >>> top_suite.find("group").find("scheme").text + 'my_scheme' + >>> xml = f''' + ... + ... + ... + ... + ... + ... ''' + >>> top_suite = ET.fromstring(xml) + >>> top_group = top_suite.find("group") + >>> nested = top_group.find("nested_suite") + >>> replace_nested_suite(top_group, nested, tmpdir.name, logger) + 'my_suite' + >>> [child.tag for child in top_suite] + ['group'] + >>> top_suite.find("group").find("scheme").text + 'my_scheme' + >>> xml = f''' + ... + ... + ... + ... ''' + >>> top_suite = ET.fromstring(xml) + >>> nested = top_suite.find("nested_suite") + >>> replace_nested_suite(top_suite, nested, tmpdir.name, logger) + 'my_suite' + >>> [child.tag for child in top_suite] + ['group'] + >>> top_suite.find("group").find("scheme").text + 'my_scheme' + >>> tmpdir.cleanup() + """ + suite_name = nested_suite.attrib.get("name") + group_name = nested_suite.attrib.get("group") + file = nested_suite.attrib.get("file") + if not os.path.isabs(file): + file = os.path.join(default_path, file) + referenced_suite = load_suite_by_name(suite_name, group_name, file, + logger=logger) + imported_content = [ET.fromstring(ET.tostring(child)) + for child in referenced_suite] + for item in imported_content: + # When importing a single group at the suite level, wrap the item + # in a fresh so the parent's level isn't changed. + if element.tag == 'suite' and group_name: + item_to_insert = ET.Element("group", attrib={"name": group_name}) + item_to_insert.append(item) + else: + item_to_insert = item + element.insert(list(element).index(nested_suite), item_to_insert) + element.remove(nested_suite) + if logger: + msg = f"Expanded nested suite '{suite_name}'" \ + + (f", group '{group_name}'," if group_name else "") \ + + (f" in file '{file}'" if file else "") + logger.debug(msg.rstrip(',')) + return suite_name + + +def expand_nested_suites(suite, default_path, logger=None): + """Recursively expand every ```` element inside *suite*. + + Iterative bound caps the recursion at ``max_iterations`` passes to + keep mutually-referential SDFs from looping forever. + + Examples + -------- + >>> import tempfile + >>> import xml.etree.ElementTree as ET + >>> logger = init_log('xml_tools') + >>> set_log_to_null(logger) + >>> tmpdir = tempfile.TemporaryDirectory() + >>> file1_path = os.path.join(tmpdir.name, "file1.xml") + >>> file2_path = os.path.join(tmpdir.name, "file2.xml") + >>> file3_path = os.path.join(tmpdir.name, "file3.xml") + >>> file4_path = os.path.join(tmpdir.name, "file4.xml") + >>> file5_path = os.path.join(tmpdir.name, "file5.xml") + >>> with open(file1_path, "w") as f: + ... _ = f.write(''' + ... + ... + ... cloud_scheme + ... + ... + ... ''') + >>> with open(file2_path, "w") as f: + ... _ = f.write(''' + ... + ... + ... pbl_scheme + ... + ... + ... ''') + >>> with open(file3_path, "w") as f: + ... _ = f.write(''' + ... + ... + ... rrtmg_lw_scheme + ... + ... + ... rrtmg_sw_scheme + ... + ... + ... ''') + >>> with open(file4_path, "w") as f: + ... _ = f.write(f''' + ... + ... + ... + ... ''') + >>> with open(file5_path, "w") as f: + ... _ = f.write(f''' + ... + ... + ... + ... ''') + >>> xml_content = f''' + ... + ... + ... + ... + ... + ... + ... + ... ''' + >>> suite = ET.fromstring(xml_content) + >>> expand_nested_suites(suite, tmpdir.name, logger) + >>> ET.dump(suite) + + + cloud_scheme + + pbl_scheme + + rrtmg_lw_scheme + + rrtmg_sw_scheme + + >>> xml_content = f''' + ... + ... + ... + ... + ... + ... + ... ''' + >>> suite = ET.fromstring(xml_content) + >>> expand_nested_suites(suite, tmpdir.name, logger) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + CCPPError: Exceeded number of iterations while expanding nested suites + >>> tmpdir.cleanup() + """ + max_iterations = 10 + suite_names = [] + for _ in range(max_iterations): + keep_expanding = False + for group in suite.findall("group"): + for nested in group.findall("nested_suite"): + suite_names.append( + replace_nested_suite(group, nested, default_path, logger)) + keep_expanding = True + for nested in suite.findall("nested_suite"): + suite_names.append( + replace_nested_suite(suite, nested, default_path, logger)) + keep_expanding = True + if not keep_expanding: + return + raise CCPPError( + "Exceeded number of iterations while expanding nested suites. " + "Check for infinite recursion or adjust limit max_iterations. " + f"Suites expanded so far: {suite_names}") + + +def write_xml_file(root, file_path, logger=None): + """Pretty-print *root* to *file_path*, routed through write_if_changed. + + Unchanged regenerations preserve the on-disk mtime so downstream + build tools don't rebuild. + """ + + def remove_whitespace_nodes(node): + for child in list(node.childNodes): + if child.nodeType == child.TEXT_NODE and not child.data.strip(): + node.removeChild(child) + elif child.hasChildNodes(): + remove_whitespace_nodes(child) + + byte_string = ET.tostring(root, 'us-ascii') + reparsed = xml.dom.minidom.parseString(byte_string) + remove_whitespace_nodes(reparsed) + pretty_xml = reparsed.toprettyxml(indent=" ") + + from .io_helpers import write_if_changed + write_if_changed(file_path, pretty_xml, logger=logger) diff --git a/capgen/metadata/registered_dimensions.py b/capgen/metadata/registered_dimensions.py new file mode 100644 index 00000000..eae837a1 --- /dev/null +++ b/capgen/metadata/registered_dimensions.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 + +"""Registered scalar-index dimensions for capgen. + +This module is the **single source of truth** for the small set of CCPP +standard-name dimensions that capgen treats specially: each one is a +*count* (e.g. ``number_of_instances``, ``number_of_threads``) whose access +pattern collapses to a paired scalar *index* variable (e.g. +``instance_number``, ``thread_number``). + +Where this is used +------------------ + +A container DDT-instance variable in host metadata may carry one of these +dimensions, e.g.:: + + [Interstitial] + standard_name = GFS_interstitial_type_instance + type = GFS_interstitial_type + dimensions = (number_of_threads) # <-- registered scalar dim + +When a scheme reaches into a field of that container, capgen emits the +scalar index automatically:: + + physics%Interstitial(thread_number)%alpha(lb:ub, 1:nlev) + ^^^^^^^^^^^^^^^ + substituted from the registered scalar-index pair + +The same machinery applies anywhere capgen needs to subscript a +container by an index variable that the host carries as a separate +control/host variable. + +The two rules +------------- + +**Rule 1 (generalized, NOT enforced as a hard gate)**: +A container DDT-instance variable may carry registered scalar-index +dimensions in its ``dimensions`` clause. When it does, capgen emits +the paired index variable's local Fortran name at every call site that +reaches into the container. Anything *not* in :data:`SCALAR_INDEX_DIMS` +flows through the normal slice/bounds machinery +(``horizontal_loop_begin:horizontal_loop_end``, ``1:vertical_*``, …) +exactly like flat-array dims. + +**Rule 2 (ENFORCED at host-dict-build time)**: +A **leaf** variable (intrinsic-typed or ``external:`` Fortran type — the +kind a physics scheme actually binds to) MUST NOT declare a registered +scalar-index dim. Leaves see only spatial / tracer / count dims; scalar +indexing is a container-DDT concept. Violations raise a ``CCPPError`` +at parse time that names the offending variable, the offending dim, the +expected index variable, and points the user back to this file. + +Why both rules matter +--------------------- + +* Rule 1 generalization lets capgen support multi-instance + (``number_of_instances``) **and** per-thread (``number_of_threads``) + container DDTs from one mechanism — no per-dimension code path. +* Rule 2 keeps the substitution mechanism contained: the rule "leaves + never carry registered scalar dims" means the scheme-call-site dim + handler (``_one_dim_part`` in ``generator/suite_resolver.py``) never + has to special-case scalar-index dims. One code path for leaves, one + code path for containers. + +How to extend the table +----------------------- + +Adding a new pairing (e.g. a new ``number_of_blocks`` ↔ ``block_number`` +convention) is a four-step process: + +1. **Add an entry below** mapping the *count* standard name to the + *index* standard name. Both are CCPP standard names the host must + declare. +2. **Verify the host metadata declares both**: a ``[ccpp-table-properties]`` + block with ``type = control`` (or ``type = host``) carrying a scalar + integer with the index standard name, plus a similar declaration for + the count. +3. **Add a unit test** to ``unit-tests/test_registered_dimensions.py`` + exercising the new pairing. +4. **Update** ``doc/migration.md`` §3 "Registered scalar-index + dimensions" and ``doc/redesign_prompt.md`` §4.3. + +The contract for users (host model authors) +------------------------------------------- + +If you see the error message:: + + Variable '' (standard_name='') declares dimension '' + on a leaf-data variable, but '' is a registered scalar-index + dimension reserved for DDT-instance containers (see + capgen/metadata/registered_dimensions.py). + [...] + +it means you wrote something like:: + + [my_array] + standard_name = some_leaf_quantity + type = real | kind = kind_phys + dimensions = (number_of_threads, horizontal_dimension) + +The fix is to **wrap** the leaf in a per-thread container DDT, e.g.:: + + [Interstitial] + type = my_interstitial_type + dimensions = (number_of_threads) + +with ``my_interstitial_type`` declaring the leaf with only its spatial +dims:: + + [some_leaf_quantity] + type = real | kind = kind_phys + dimensions = (horizontal_dimension) + +This mirrors how every CCPP host in production today (UFS, NEPTUNE, +CAM-SIMA, ccpp-scm) structures per-thread / per-instance state. +""" + +from typing import Dict, FrozenSet, Optional + + +######################################################################## +# The registered scalar-index dimension table +######################################################################## + +#: Map from *count* dimension standard name → paired *index* variable +#: standard name. The host metadata MUST declare the index variable as +#: a scalar integer (in a ``type = control`` table when it is a +#: framework-lifecycle variable, or in a ``type = host`` table +#: otherwise). +#: +#: Every entry here is treated as a hard convention across the entire +#: CCPP ecosystem. Adding an entry binds capgen to a specific +#: standard-name pairing; once an entry lands and hosts adopt it, +#: removing or renaming it is a breaking change. +SCALAR_INDEX_DIMS: Dict[str, str] = { + # Multi-instance API: the framework's instance_number paired opt-in. + # Hosts that declare instance_number + number_of_instances opt into + # the multi-instance API; capgen auto-substitutes (instance_number) + # wherever a container DDT carries this dimension. + 'number_of_instances': 'instance_number', + + # Per-thread DDT containers (e.g. ``physics%Interstitial(thread_number)``) + # — the host's openmp-thread index. (thread_number, number_of_threads) is + # a paired-optional control pair (see ccpp_capgen._PAIRED_OPTIONAL_CTRL_VARS + # and doc/migration.md §3.1). A host that dimensions a variable by + # ``number_of_threads`` MUST declare the pair — otherwise the collapse + # below cannot find ``thread_number`` and raises. + 'number_of_threads': 'thread_number', +} + + +######################################################################## +# Public helpers +######################################################################## + +def scalar_index_for(dim_std_name: str) -> Optional[str]: + """Return the paired scalar-index std name for *dim_std_name*, or None. + + >>> scalar_index_for('number_of_instances') + 'instance_number' + >>> scalar_index_for('number_of_threads') + 'thread_number' + >>> scalar_index_for('horizontal_dimension') is None + True + """ + return SCALAR_INDEX_DIMS.get(dim_std_name) + + +def is_scalar_index_dim(dim_std_name: str) -> bool: + """Return True iff *dim_std_name* is a registered scalar-index dimension. + + >>> is_scalar_index_dim('number_of_instances') + True + >>> is_scalar_index_dim('horizontal_dimension') + False + """ + return dim_std_name in SCALAR_INDEX_DIMS + + +def registered_count_dims() -> FrozenSet[str]: + """Return the set of all registered count-side dimension std names. + + Equivalent to ``frozenset(SCALAR_INDEX_DIMS)``; provided as a helper + so consumers don't have to reach into the dict. + """ + return frozenset(SCALAR_INDEX_DIMS) + + +def registered_index_vars() -> FrozenSet[str]: + """Return the set of all registered scalar-index variable std names. + + Equivalent to ``frozenset(SCALAR_INDEX_DIMS.values())``. + """ + return frozenset(SCALAR_INDEX_DIMS.values()) diff --git a/scripts/conversion_tools/unit_conversion.py b/capgen/metadata/unit_conversion.py similarity index 100% rename from scripts/conversion_tools/unit_conversion.py rename to capgen/metadata/unit_conversion.py diff --git a/capgen/metadata/variable_resolver.py b/capgen/metadata/variable_resolver.py new file mode 100644 index 00000000..82cd7236 --- /dev/null +++ b/capgen/metadata/variable_resolver.py @@ -0,0 +1,937 @@ +#!/usr/bin/env python3 + +"""Variable resolution and access-path construction for ccpp-capgen. + +This module flattens the host/control/DDT metadata hierarchy into a single +keyed dictionary and provides the ``SchemeStore`` lookup table that the code +generator uses to resolve scheme arguments. + +Public API +---------- +``HostVarEntry`` + One resolved variable in the flat host+control dictionary. + +``build_flat_host_dict(host_tables, control_tables, ddt_tables)`` + Flatten host + control tables (expanding DDT instances into their fields) + into a ``Dict[str, HostVarEntry]`` keyed by standard name. + +``SchemeStore`` + Organises scheme metadata by scheme name and phase for O(1) lookup during + variable resolution. + +Registered dimension standard names (Section 4.3 of the redesign spec) +------------------------------------------------------------------------ +The generator has built-in semantic knowledge of these standard names when they +appear as dimensions in host variables: + +``instance_dimension`` + Scalar extraction — the DDT instance array is subscripted with the + ``instance_number`` control variable: + ``gfs_statein(instance_number)%field`` + + Note: the sample metadata in section 3.5 of the spec uses + ``number_of_instances`` for the same role. Both names are treated as + instance dimensions here; see ``_INSTANCE_DIMS``. + +``horizontal_dimension`` + Horizontal slice — emitted as ``lb:ub`` (run phase) or + ``1:`` (non-run). The code generator handles the slicing; + the resolver only records the dimension standard name. + +``vertical_*`` + Vertical slice — emitted as ``1:``. Same handling. + +All other dimension standard names are "arbitrary" — the resolver looks up +the corresponding variable's local name and the code generator emits +``1:``. +""" + +import re +from typing import Dict, List, Optional + +from .metadata_table import MetaVar, MetadataTable +from .parse_tools import CCPPError, check_fortran_intrinsic, FORTRAN_SCALAR_REF_RE + +######################################################################## +# Registered dimension constants +######################################################################## + +# The set of registered scalar-index dimensions (e.g. +# ``number_of_instances`` → ``instance_number``, +# ``number_of_threads`` → ``thread_number``) lives in a single +# documented module so the contract is easy for users and developers to +# find and extend. See ``capgen/metadata/registered_dimensions.py`` +# for the full table and the two rules that govern it. +from .registered_dimensions import ( + SCALAR_INDEX_DIMS, + scalar_index_for, + is_scalar_index_dim, + registered_count_dims, +) + +#: Regex that normalises ``type(typename)`` → ``typename``. +_TYPE_PAREN_RE = re.compile( + r'(?i)^type\s*\(\s*([A-Za-z][A-Za-z0-9_]*)\s*\)$' +) + +#: Regex for ``external:module:typename`` type syntax. +_EXTERNAL_RE = re.compile(r'^external\s*:', re.IGNORECASE) + +######################################################################## +# Helpers +######################################################################## + +def _ddt_typename(type_str: str) -> str: + """Return the bare DDT type name, stripping ``type(...)`` if present.""" + m = _TYPE_PAREN_RE.match(type_str.strip()) + return m.group(1) if m else type_str.strip() + + +def _is_intrinsic(type_str: str) -> bool: + """Return True if *type_str* is a Fortran intrinsic type name.""" + return check_fortran_intrinsic(type_str.strip(), error=False) is not None + + +def _is_external(type_str: str) -> bool: + """Return True if *type_str* uses the ``external:module:typename`` syntax.""" + return bool(_EXTERNAL_RE.match(type_str.strip())) + + +def _is_known_ddt(type_str: str, ddt_index: Dict[str, MetadataTable]) -> bool: + """Return True if *type_str* refers to a DDT type present in *ddt_index*.""" + if _is_intrinsic(type_str) or _is_external(type_str): + return False + return _ddt_typename(type_str) in ddt_index + + +def _split_local_name(local_name: str): + """Split a local name into (base, subscript) tuple. + + For plain identifiers returns (local_name, ''). + For slice expressions like ``field(idx)`` returns + (``'field'``, ``'idx'``). + + >>> _split_local_name('field') + ('field', '') + >>> _split_local_name('field(idx)') + ('field', 'idx') + >>> _split_local_name('q(:,:,index_of_water_vapor_specific_humidity)') + ('q', ':,:,index_of_water_vapor_specific_humidity') + """ + m = FORTRAN_SCALAR_REF_RE.match(local_name) + if m is None: + return local_name, '' + return m.group(1), m.group(2).rstrip() + + +def _resolve_subscript(subscript: str, host_dict: Dict[str, 'HostVarEntry']) -> str: + """Replace each standard-name token in *subscript* with its local name. + + Tokens that are ``:`` (colon slices) or integer literals are left as-is. + Tokens that are standard names present in *host_dict* are replaced by + the entry's ``local_name``. Unrecognised tokens are left as-is (they + may be integer constants or already-local names). + + >>> from collections import namedtuple + >>> E = namedtuple('E', ['local_name']) + >>> d = {'thread_number': E('thrd_no')} + >>> _resolve_subscript('thread_number', d) + 'thrd_no' + >>> _resolve_subscript(':, thread_number', d) + ':, thrd_no' + >>> _resolve_subscript('1', d) + '1' + """ + tokens = [t.strip() for t in subscript.split(',')] + resolved = [] + for token in tokens: + if token == ':' or token.isdigit(): + resolved.append(token) + elif token in host_dict: + resolved.append(host_dict[token].local_name) + else: + resolved.append(token) + return ', '.join(resolved) + + +def _validate_leaf_dims(var: 'MetaVar', source_label: str) -> None: + """Reject leaf variables that declare a registered scalar-index dim. + + Rule 2 of the registered-scalar-index-dimension contract (see + :mod:`metadata.registered_dimensions`): a *leaf* variable — one that + a physics scheme actually binds to (intrinsic-typed or ``external:`` + Fortran type) — MUST NOT declare a dim like ``number_of_instances`` + or ``number_of_threads``. Those dims belong on container + DDT-instance variables in the access path, never on the leaf data + itself. + + Raises + ------ + CCPPError + With a message that names the offending variable, the offending + dim, the paired index variable, the source label (host file / + DDT table where the leaf was declared), and a pointer back to + :mod:`metadata.registered_dimensions` for the full table and + the remediation pattern. + """ + offenders = [d for d in var.dimensions if is_scalar_index_dim(d)] + if not offenders: + return + dim = offenders[0] + idx_std = scalar_index_for(dim) + raise CCPPError( + "Variable '{name}' (standard_name='{std}', declared in {src}) " + "is a leaf data variable but its dimensions list includes " + "'{dim}', a registered scalar-index dimension reserved for " + "DDT-instance container variables (paired with index " + "'{idx}').\n" + "\n" + "Leaf variables (intrinsic- or external-typed, the kind a " + "physics scheme binds to) MUST NOT carry registered scalar-" + "index dimensions. Wrap '{name}' in a container DDT whose " + "dimensions = ({dim}), and declare '{name}' inside that DDT " + "with only its spatial / tracer / count dims. The generator " + "will emit '({idx})%{name}(...)' at every scheme " + "call site automatically.\n" + "\n" + "See capgen/metadata/registered_dimensions.py for the full " + "table of registered scalar-index pairings and how to extend " + "it.".format( + name=var.local_name, + std=var.standard_name, + src=source_label, + dim=dim, + idx=idx_std, + ) + ) + + +def _instance_subscript(var: MetaVar) -> str: + """Return the scalar-index subscript for a container DDT-instance variable. + + Walks *var*'s declared dimensions in order; for each dim that is a + registered scalar-index dim (see + :mod:`metadata.registered_dimensions`), emits the paired index + variable's standard name as a placeholder. The placeholder is + resolved to the host's local Fortran name at codegen time by + :func:`generator.suite_resolver._substitute_scalar_idx`. + + Returns + ------- + str + Subscript string such as ``'(instance_number)'``, + ``'(thread_number)'``, or for multi-pair containers + ``'(instance_number, thread_number)'`` — one component per + registered scalar-index dim found in *var.dimensions* in + declared order. Returns ``''`` when no registered scalar dim + is present (the caller is left to handle non-registered dims + through the normal slice machinery). + """ + parts = [] + for dim in var.dimensions: + idx = scalar_index_for(dim) + if idx is not None: + parts.append(idx) + if not parts: + return '' + return '({})'.format(', '.join(parts)) + + +######################################################################## +# HostVarEntry +######################################################################## + +class HostVarEntry: + """One resolved variable in the flat host+control dictionary. + + All fields are set at construction time and treated as read-only + afterwards. + + Parameters + ---------- + standard_name : str + CF-compliant standard name (key in the flat dict). + local_name : str + Fortran local name of the innermost variable (e.g. ``'phii'``). + access_path : str + Fully-qualified Fortran access expression, with any DDT + component separators and instance subscripts applied + (e.g. ``'gfs_statein(instance_number)%phii'``). For plain + variables this equals *local_name*. + module_name : str or None + Fortran module that exports this variable, used to emit + ``use , only: `` in the generated cap. + ``None`` for control variables (passed as subroutine arguments). + type : str + Fortran type string. + kind : str + Optional kind parameter (empty string if not specified). + units : str + Physical units. + dimensions : list of str + Ordered dimension standard names; empty for scalars. + protected : bool + Whether any scheme is forbidden from declaring ``intent`` other + than ``in`` for this variable. + optional : bool + Whether the variable may be absent (uses optional pointer in cap). + allocatable : bool + Whether the variable is declared with the Fortran ``allocatable`` + attribute. Host and scheme metadata must agree. Affects code + generation: actual arguments at call sites omit explicit dimension + subscripts for allocatable variables. + active : str + Fortran conditional expression in standard names; empty if always + active. + """ + + __slots__ = ( + 'standard_name', 'local_name', 'access_path', 'module_name', + 'type', 'kind', 'units', 'dimensions', + 'protected', 'optional', 'allocatable', 'active', 'local_subscript', + 'top_at_one', + ) + + def __init__( + self, + standard_name: str, + local_name: str, + access_path: str, + module_name: Optional[str], + type_: str, + kind: str, + units: str, + dimensions: List[str], + protected: bool, + optional: bool, + active: str, + local_subscript: Optional[List[str]] = None, + allocatable: bool = False, + top_at_one: bool = False, + ): + self.standard_name = standard_name + self.local_name = local_name + self.access_path = access_path + self.module_name = module_name + self.type = type_ + self.kind = kind + self.units = units + self.dimensions = list(dimensions) + self.protected = protected + self.optional = optional + self.allocatable = allocatable + self.active = active + self.local_subscript = list(local_subscript) if local_subscript else [] + self.top_at_one = top_at_one + + @property + def is_control(self) -> bool: + """True when this variable is a control variable (no module USE needed).""" + return self.module_name is None + + def __repr__(self) -> str: + return "HostVarEntry({!r}, access_path={!r})".format( + self.standard_name, self.access_path + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, HostVarEntry): + return NotImplemented + return self.standard_name == other.standard_name + + def __hash__(self) -> int: + return hash(self.standard_name) + + +######################################################################## +# DDT index +######################################################################## + +def _build_ddt_index(ddt_tables: List[MetadataTable]) -> Dict[str, MetadataTable]: + """Build a dict from DDT type name → ``MetadataTable`` for O(1) lookup.""" + return {tbl.table_name: tbl for tbl in ddt_tables} + + +def _resolve_module_name(tbl: MetadataTable) -> str: + """Return the Fortran module that exports *tbl*'s symbols. + + Honors the per-table ``module_name = …`` override from + ``[ccpp-table-properties]`` when present (the + ``design_module_name_override`` rule); otherwise falls back to the + table name (the implicit "module name = table name" convention). + """ + return (tbl.module_name or '').strip() or tbl.table_name + + +def build_ddt_module_map( + all_tables: List[MetadataTable], +) -> Dict[str, str]: + """Build a map from DDT type name → Fortran module that defines it. + + Resolution order, per DDT table: + + 1. **DDT's own override.** If the DDT's own ``[ccpp-table-properties]`` + carries ``module_name = …``, that wins. Most specific source — a + DDT may genuinely live in a different Fortran module than the + scheme/host its ``.meta`` is paired with. Required when the DDT + lives in a file with no co-located scheme/host/control table at + all (real-world example: CCPP-physics + ``Radiation/RRTMG/radsw_param.meta`` declares ``cmpfsw_type`` in + Fortran ``module module_radsw_parameters``, with no co-located + scheme metadata). + 2. **Co-located table's resolved module.** Failing the DDT's own + override, inherit from a co-located ``host``, ``control``, or + ``scheme`` table in the same ``.meta`` file. Its module is + resolved by the same rule used elsewhere in capgen + (:func:`_resolve_module_name`): the co-located table's own + ``module_name = …`` if declared, else its table name. + + DDT tables that pass neither rule are skipped (no entry written). + Those DDTs are only safe to leave out when no generator output + references them directly — e.g. a DDT referenced only as the type + of a host instance variable, where the host's own Fortran already + imports the type. + + What happens when both are present + ---------------------------------- + + +-------------------------+-------------------------+-----------------+ + | DDT ``module_name=`` | Co-located ``module_ | Result | + | | name=`` (or table name) | | + +=========================+=========================+=================+ + | X (set) | Y (set or default) | X — DDT wins | + +-------------------------+-------------------------+-----------------+ + | unset | Y (set or default) | Y | + +-------------------------+-------------------------+-----------------+ + | X (set) | (no co-located table) | X | + +-------------------------+-------------------------+-----------------+ + | unset | (no co-located table) | (skipped) | + +-------------------------+-------------------------+-----------------+ + + Parameters + ---------- + all_tables : list of MetadataTable + Mixed list of all parsed metadata tables (host, control, scheme, + ddt). ``suite`` tables are ignored. + + Returns + ------- + dict mapping DDT type name → Fortran module name + """ + by_file: Dict[str, List[MetadataTable]] = {} + for tbl in all_tables: + by_file.setdefault(tbl.file_path, []).append(tbl) + + result: Dict[str, str] = {} + for fpath, tables in by_file.items(): + # Co-located non-DDT table provides the fallback module name. + # Apply the same module_name-override-then-table-name resolution + # that ``build_flat_host_dict`` uses so a host/scheme that + # carries ``module_name = X`` is honored consistently. + colocated_module: Optional[str] = None + for tbl in tables: + if tbl.table_type in ('scheme', 'host', 'control'): + colocated_module = _resolve_module_name(tbl) + break + + for tbl in tables: + if tbl.table_type != 'ddt': + continue + # Per-table explicit override on the DDT itself wins. + if tbl.module_name: + result[tbl.table_name] = tbl.module_name + continue + if colocated_module is not None: + result[tbl.table_name] = colocated_module + return result + + +######################################################################## +# DDT instance flattening +######################################################################## + +def _flatten_ddt_instance( + var: MetaVar, + module_name: str, + ddt_index: Dict[str, MetadataTable], + access_prefix: str = '', + depth: int = 0, + max_depth: int = 8, +) -> List[HostVarEntry]: + """Expand a DDT instance variable into flat ``HostVarEntry`` objects. + + The DDT instance entry itself is included first, followed by one entry + per field. If a field is itself a known DDT type, it is expanded + recursively (up to *max_depth* levels deep). + + Parameters + ---------- + var : MetaVar + The DDT instance variable from a host table section. + module_name : str + Fortran module from which the DDT instance is imported. + ddt_index : dict + Map from DDT type name → ``MetadataTable``, built by + :func:`_build_ddt_index`. + access_prefix : str + Fortran access-path prefix accumulated from enclosing DDT levels, + e.g. ``'outer(instance_number)%'``. + depth, max_depth : int + Recursion guard — raises ``CCPPError`` if exceeded. + + Returns + ------- + list of HostVarEntry + + Raises + ------ + CCPPError + If the DDT type is not in *ddt_index*, or if the nesting depth + exceeds *max_depth* (circular reference guard). + """ + if depth > max_depth: + raise CCPPError( + "DDT hierarchy for '{}' exceeds maximum nesting depth {}; " + "possible circular reference".format(var.standard_name, max_depth) + ) + + ddt_name = _ddt_typename(var.type) + if ddt_name not in ddt_index: + raise CCPPError( + "Variable '{}' (standard_name='{}') has type '{}' but no " + "matching 'type = ddt' table was found; " + "add the DDT metadata file to --host-files or --scheme-files".format( + var.local_name, var.standard_name, var.type + ) + ) + + ddt_table = ddt_index[ddt_name] + subscript = _instance_subscript(var) + # If the DDT instance has dimensions but NONE of them are a + # registered scalar-index dim, capgen can't bake a meaningful + # scalar subscript into field access paths. Two outcomes are both + # legitimate, depending on how schemes use this DDT: + # + # (a) Schemes take the whole sliced DDT array as a single arg + # (e.g. ``call rad_lw_run(fluxLW=phys_state%fluxLW(lb:ub), …)``) + # and dereference inner fields inside the scheme. Flattening + # this DDT's components into host_dict is wasted and emits + # Fortran the compiler rejects. + # (b) Schemes request individual inner fields by standard name, + # which would require ``parent%var()%field(…)`` access + # with a meaningful ```` capgen can't synthesize. + # + # Skip the recursion either way: the DDT-instance's own entry is + # still recorded (case (a) just works), and case (b) trips the + # resolver's existing "standard_name not found" error when a scheme + # tries to use a would-have-been-flattened inner field. Use + # ``--legacy-mode`` (or fix the host metadata) when the underlying + # cause is a deprecated dimension name like + # ``number_of_openmp_threads``. + skip_recurse = bool(var.dimensions) and not subscript + # Fortran access path to this DDT instance (without field component). + instance_access = access_prefix + var.local_name + subscript + + entries: List[HostVarEntry] = [] + + # The DDT instance variable itself — keyed by its own standard_name. + entries.append(HostVarEntry( + standard_name=var.standard_name, + local_name=var.local_name, + access_path=access_prefix + var.local_name, + module_name=module_name, + type_=var.type, + kind=var.kind, + units=var.units, + dimensions=var.dimensions, + protected=var.protected, + optional=var.optional, + active=var.active, + local_subscript=[], + allocatable=var.allocatable, + top_at_one=var.top_at_one, + )) + + # When the DDT-instance carries non-registered dims (skip_recurse), + # leave its fields un-flattened — only the DDT-instance entry above + # is recorded. Schemes taking the whole sliced DDT work via that + # entry; schemes asking for inner fields by std_name trip the + # resolver's standard "not found" error. + if skip_recurse: + return entries + + # Expand each field of the DDT. + for sec in ddt_table.sections(): + for field in sec.variables: + if _is_known_ddt(field.type, ddt_index): + # Nested DDT — recurse. + entries.extend(_flatten_ddt_instance( + field, + module_name, + ddt_index, + access_prefix=instance_access + '%', + depth=depth + 1, + max_depth=max_depth, + )) + else: + # Rule 2: a leaf DDT field cannot carry a registered + # scalar-index dim. Surface the violation at parse time + # with a clear remediation pointer. + _validate_leaf_dims( + field, + "DDT '{}' (file: {})".format( + ddt_name, + ddt_table.file_path, + ), + ) + base_field, sub_str = _split_local_name(field.local_name) + sub_tokens = [t.strip() for t in sub_str.split(',') if t.strip()] if sub_str else [] + field_path = instance_access + '%' + base_field + entries.append(HostVarEntry( + standard_name=field.standard_name, + local_name=base_field, + access_path=field_path, + module_name=module_name, + type_=field.type, + kind=field.kind, + units=field.units, + dimensions=field.dimensions, + protected=field.protected, + optional=field.optional, + active=field.active, + local_subscript=sub_tokens, + allocatable=field.allocatable, + top_at_one=field.top_at_one, + )) + + return entries + + +######################################################################## +# Public: build_flat_host_dict +######################################################################## + +def build_flat_host_dict( + host_tables: List[MetadataTable], + control_tables: List[MetadataTable], + ddt_tables: List[MetadataTable], +) -> Dict[str, 'HostVarEntry']: + """Build the flat host+control variable dictionary. + + All host and control variables are flattened into a single + ``Dict[str, HostVarEntry]`` keyed by standard name. DDT instance + variables are expanded into one entry per field; the instance variable + itself is also stored under its own standard name. + + Control variables have ``module_name=None`` — they are passed as + subroutine arguments, not imported via ``use``. + + Parameters + ---------- + host_tables : list of MetadataTable + Tables with ``table_type == 'host'``. + control_tables : list of MetadataTable + Tables with ``table_type == 'control'``. + ddt_tables : list of MetadataTable + Tables with ``table_type == 'ddt'``. + + Returns + ------- + dict mapping standard_name → HostVarEntry + + Raises + ------ + CCPPError + On duplicate standard names across tables, or on a DDT reference + without a matching DDT table. + """ + ddt_index = _build_ddt_index(ddt_tables) + result: Dict[str, HostVarEntry] = {} + + def _add(entry: HostVarEntry, source_label: str) -> None: + # A character variable whose storage is DEFINED by host or DDT + # metadata must be concrete: ``len=*`` (assumed length) is illegal + # for a host module variable or a derived-type component. Control + # variables are EXEMPT -- they are pass-through dummy arguments + # (suite_name, errmsg, ...) which the generated caps legitimately + # declare ``character(len=*)``. Reject it here so the error names + # the table rather than surfacing downstream as undeclarable Fortran. + if (not entry.is_control + and (entry.type or '').strip().lower() == 'character' + and (entry.kind or '').strip() == 'len=*'): + raise CCPPError( + "Character variable '{}' (standard_name='{}') in table '{}' " + "declares kind='len=*'; host and DDT metadata must give " + "character variables a concrete length (e.g. kind=len=512) " + "-- assumed length is valid only for dummy arguments (scheme " + "args and control/lifecycle variables).".format( + entry.local_name, entry.standard_name, source_label, + ) + ) + prior = result.get(entry.standard_name) + if prior is not None: + prior_loc = ( + "module '{}'".format(prior.module_name) + if prior.module_name else '' + ) + new_loc = "module '{}'".format(entry.module_name) \ + if entry.module_name else '' + raise CCPPError( + "Duplicate standard name '{}':\n" + " already registered from {} via access path '{}'\n" + " re-registered from {} (source: {}) via access path '{}'\n" + "If both paths come from the same parent DDT, the parent " + "likely declares two sibling DDT instances of the same " + "type — components of each get flattened into the host " + "dictionary under the same standard names. Drop one of " + "the sibling DDT instances, or give one a non-overlapping " + "standard name on every component.".format( + entry.standard_name, + prior_loc, prior.access_path, + new_loc, source_label, entry.access_path, + ) + ) + result[entry.standard_name] = entry + + # ---- host tables ------------------------------------------------------- + for tbl in host_tables: + # Explicit ``module_name`` from ``[ccpp-table-properties]`` overrides + # the convention "module name = table name"; falls back to the table + # name when not declared. + host_module = (tbl.module_name or '').strip() or tbl.table_name + for sec in tbl.sections(): + for var in sec.variables: + if _is_known_ddt(var.type, ddt_index): + for entry in _flatten_ddt_instance( + var, host_module, ddt_index + ): + _add(entry, tbl.table_name) + elif _is_intrinsic(var.type) or _is_external(var.type): + _validate_leaf_dims( + var, + "host table '{}' (file: {})".format( + tbl.table_name, tbl.file_path, + ), + ) + base_name, sub_str = _split_local_name(var.local_name) + sub_tokens = [t.strip() for t in sub_str.split(',') if t.strip()] if sub_str else [] + _add(HostVarEntry( + standard_name=var.standard_name, + local_name=base_name, + access_path=base_name, + module_name=host_module, + type_=var.type, + kind=var.kind, + units=var.units, + dimensions=var.dimensions, + protected=var.protected, + optional=var.optional, + active=var.active, + local_subscript=sub_tokens, + allocatable=var.allocatable, + top_at_one=var.top_at_one, + ), tbl.table_name) + else: + raise CCPPError( + "Variable '{}' (standard_name='{}') in table '{}' has " + "type '{}' which is not a Fortran intrinsic, not an " + "'external:' type, and has no matching 'type = ddt' table; " + "declare the DDT in a metadata file and pass it via " + "--host-files, or use 'type = external::' " + "for non-CCPP types".format( + var.local_name, var.standard_name, + tbl.table_name, var.type, + ) + ) + + # ---- control tables ---------------------------------------------------- + for tbl in control_tables: + for sec in tbl.sections(): + for var in sec.variables: + _validate_leaf_dims( + var, + "control table '{}' (file: {})".format( + tbl.table_name, tbl.file_path, + ), + ) + base_name, sub_str = _split_local_name(var.local_name) + sub_tokens = [t.strip() for t in sub_str.split(',') if t.strip()] if sub_str else [] + _add(HostVarEntry( + standard_name=var.standard_name, + local_name=base_name, + access_path=base_name, + module_name=None, + type_=var.type, + kind=var.kind, + units=var.units, + dimensions=var.dimensions, + protected=var.protected, + optional=var.optional, + active=var.active, + local_subscript=sub_tokens, + allocatable=var.allocatable, + top_at_one=var.top_at_one, + ), tbl.table_name) + + return result + + +######################################################################## +# SchemeStore +######################################################################## + +class SchemeStore: + """Organises scheme metadata for variable resolution. + + Provides O(1) lookup of scheme arguments by scheme name and phase. + Constructed via :meth:`build_from`. + """ + + def __init__(self) -> None: + # _data[scheme_name][phase] = list of MetaVar (in metadata order) + self._data: Dict[str, Dict[str, List[MetaVar]]] = {} + # _modules[scheme_name] = Fortran module that exports the scheme's + # subroutines. Populated from the metadata table's ``module_name`` + # attribute (``[ccpp-table-properties]``) when declared; otherwise + # falls back to the scheme name (the common case where the .meta + # file shares its base name with the Fortran module). + self._modules: Dict[str, str] = {} + # _source_paths[scheme_name][phase] = .meta file that first + # registered this (scheme, phase) pair. Consulted only on the + # duplicate-phase error path so the message can name both the + # original registration site and the duplicate. + self._source_paths: Dict[str, Dict[str, str]] = {} + # Memoised set of constituent standard names (base + tendency; + # see constituent_stdnames); None until first computed. + self._const_stds: Optional[frozenset] = None + + @classmethod + def build_from(cls, scheme_tables: List[MetadataTable]) -> 'SchemeStore': + """Build a :class:`SchemeStore` from *scheme_tables*. + + Non-scheme tables are silently skipped so the caller may pass a + mixed list without filtering. + + Parameters + ---------- + scheme_tables : list of MetadataTable + One or more metadata tables (only ``scheme`` tables are used). + + Returns + ------- + SchemeStore + """ + store = cls() + for tbl in scheme_tables: + if not tbl.is_scheme: + continue + name = tbl.table_name + if name not in store._data: + store._data[name] = {} + store._source_paths[name] = {} + # Resolve module: explicit ``module_name`` from the table + # properties overrides the implicit "module name equals scheme + # name" convention. See doc/scheme metadata format. + mod = tbl.module_name.strip() if tbl.module_name else '' + store._modules[name] = mod or name + for sec in tbl.sections(): + if sec.phase is None: + continue + if sec.phase in store._data[name]: + first_path = store._source_paths[name].get(sec.phase, '') + dup_path = tbl.file_path or '' + # Same-path duplicate is the common case (a .meta + # file listed twice in the host's --scheme-files + # input, often a stray CMake list entry); call it + # out so the user knows to look in the build glue + # rather than in the metadata content. + if first_path == dup_path: + hint = (' (both paths are identical — likely a ' + 'duplicate entry in the --scheme-files ' + 'list passed to capgen)') + else: + hint = '' + raise CCPPError( + "Duplicate phase '{}' for scheme '{}': " + "first registered from '{}', then again from " + "'{}'.{} Check that the same scheme metadata " + "is not loaded twice.".format( + sec.phase, name, first_path, dup_path, hint, + ) + ) + store._data[name][sec.phase] = list(sec.variables) + store._source_paths[name][sec.phase] = tbl.file_path or '' + return store + + def scheme_names(self) -> List[str]: + """Return sorted list of all known scheme names.""" + return sorted(self._data.keys()) + + def constituent_stdnames(self) -> frozenset: + """Standard names that some scheme declares as a CONSTITUENT. + + A variable is a constituent when any scheme argument flags it + ``is_constituent`` (``advected`` / ``constituent`` / ``molar_mass``): + + * a base constituent -- e.g. a mixing ratio flagged ``advected = true`` + (read via ``vars_layer``), or + * a constituent tendency -- a ``tendency_of_*`` arg flagged + ``constituent = true`` (read via ``vars_layer_tend``). + + A *consumer* of either (a scheme reading a mixing ratio, or a diagnostics + scheme reading a tendency) must NOT re-flag it: whether a given standard + name is a constituent or an ordinary variable is the host's decision + (CAM-SIMA registers it as a constituent; CCPP-SCM may expose the same + name as an ordinary host variable). The suite resolver therefore infers + constituent-ness from this scheme-metadata-wide set rather than from the + consumer's own metadata. Memoised; covers every loaded scheme because + standard-name semantics are global. + """ + if self._const_stds is None: + result = set() + for phases in self._data.values(): + for varlist in phases.values(): + for var in varlist: + if getattr(var, 'is_constituent', False): + result.add(var.standard_name) + self._const_stds = frozenset(result) + return self._const_stds + + def module_for(self, name: str) -> str: + """Return the Fortran module name that exports scheme *name*. + + Returns the explicit ``module_name`` from the scheme's + ``[ccpp-table-properties]`` block when set, falling back to the + scheme name (the common case where the .meta file's table name + equals the Fortran module name). Returns *name* unchanged for + unknown schemes so callers always get a usable token for USE + emission; whether the unknown scheme exists in Fortran is + validated elsewhere (the cap simply fails to link). + """ + return self._modules.get(name, name) + + def has_scheme(self, name: str) -> bool: + """Return True if *name* is a known scheme.""" + return name in self._data + + def phases_for(self, name: str) -> List[str]: + """Return sorted list of phases defined for scheme *name*. + + Returns an empty list for unknown schemes. + """ + return sorted(self._data.get(name, {}).keys()) + + def variables_for(self, name: str, phase: str) -> Optional[List[MetaVar]]: + """Return the variable list for *name* / *phase*, or ``None``. + + The returned list preserves the metadata declaration order, which + determines argument order in the generated scheme call. + """ + phases = self._data.get(name) + if phases is None: + return None + inner = phases.get(phase) + return list(inner) if inner is not None else None + + def __repr__(self) -> str: + return "SchemeStore(schemes={})".format(self.scheme_names()) diff --git a/schema/suite_v1_0.xsd b/capgen/schema/suite_v1_0.xsd similarity index 100% rename from schema/suite_v1_0.xsd rename to capgen/schema/suite_v1_0.xsd diff --git a/schema/suite_v2_0.xsd b/capgen/schema/suite_v2_0.xsd similarity index 91% rename from schema/suite_v2_0.xsd rename to capgen/schema/suite_v2_0.xsd index 9a69efa5..6ce3cfc6 100644 --- a/schema/suite_v2_0.xsd +++ b/capgen/schema/suite_v2_0.xsd @@ -104,18 +104,12 @@ - - - - + - - - - + diff --git a/src/ccpp_constituent_prop_mod.F90 b/capgen/src/ccpp_constituent_prop_mod.F90 similarity index 97% rename from src/ccpp_constituent_prop_mod.F90 rename to capgen/src/ccpp_constituent_prop_mod.F90 index d881e308..dbe33f84 100644 --- a/src/ccpp_constituent_prop_mod.F90 +++ b/capgen/src/ccpp_constituent_prop_mod.F90 @@ -56,6 +56,12 @@ module ccpp_constituent_prop_mod ! default_value is the default value that the constituent array will be ! initialized to real(kind=kind_phys), private :: const_default_value = kphys_unassigned + ! framework_owns_me is set by the caller of ccpp_model_constituents_t%new_field + ! (via set_framework_owned) when the caller has allocated the object on + ! the heap and is transferring ownership to the framework. When .false. + ! (default), the framework treats this object as caller-owned and will + ! not deallocate it during reset/teardown. + logical, private :: framework_owns_me = .false. contains ! Required hashable method procedure :: key => ccp_properties_get_key @@ -96,6 +102,9 @@ module ccpp_constituent_prop_mod procedure :: set_water_species => ccp_set_water_species procedure :: set_minimum => ccp_set_min_val procedure :: set_molar_mass => ccp_set_molar_mass + ! Ownership flag for framework-side cleanup (see field comment above) + procedure :: is_framework_owned => ccp_is_framework_owned + procedure :: set_framework_owned => ccp_set_framework_owned end type ccpp_constituent_properties_t !! \section arg_table_ccpp_constituent_prop_ptr_t @@ -498,11 +507,38 @@ subroutine ccp_deallocate(this) this%const_type = int_unassigned this%const_water = int_unassigned this%const_default_value = kphys_unassigned + this%framework_owns_me = .false. end subroutine ccp_deallocate !####################################################################### + logical function ccp_is_framework_owned(this) result(owned) + ! Return .true. if this object's storage was transferred to the framework + ! (via set_framework_owned) and should be deallocated during teardown. + + class(ccpp_constituent_properties_t), intent(in) :: this + + owned = this%framework_owns_me + + end function ccp_is_framework_owned + + !####################################################################### + + subroutine ccp_set_framework_owned(this, value) + ! Mark this object as owned by the framework (value=.true.) when the + ! caller has allocated it on the heap and is transferring ownership. + ! Call before passing to ccpp_model_constituents_t%new_field. + + class(ccpp_constituent_properties_t), intent(inout) :: this + logical, intent(in) :: value + + this%framework_owns_me = value + + end subroutine ccp_set_framework_owned + + !####################################################################### + subroutine ccp_get_standard_name(this, std_name, errcode, errmsg) ! Return this constituent's standard name @@ -2537,16 +2573,23 @@ end subroutine ccpt_set !######################################################################## subroutine ccpt_deallocate(this) - ! Deallocate the constituent object pointer if it is allocated. + ! Release the wrapper's reference to its constituent property object. + ! If the framework owns the object (i.e., the caller of + ! ccpp_model_constituents_t%new_field allocated it and flipped its + ! framework_owns_me flag via set_framework_owned), free its internal + ! storage and deallocate the object itself. Otherwise, the object is + ! caller-owned and we only drop our pointer to it. ! Dummy argument class(ccpp_constituent_prop_ptr_t), intent(inout) :: this if (associated(this%prop)) then - call this%prop%deallocate() - deallocate(this%prop) + if (this%prop%is_framework_owned()) then + call this%prop%deallocate() + deallocate(this%prop) + end if + nullify(this%prop) end if - nullify(this%prop) end subroutine ccpt_deallocate diff --git a/src/ccpp_constituent_prop_mod.meta b/capgen/src/ccpp_constituent_prop_mod.meta similarity index 98% rename from src/ccpp_constituent_prop_mod.meta rename to capgen/src/ccpp_constituent_prop_mod.meta index 77f446e6..657f3d8e 100644 --- a/src/ccpp_constituent_prop_mod.meta +++ b/capgen/src/ccpp_constituent_prop_mod.meta @@ -42,7 +42,6 @@ standard_name = ccpp_constituents long_name = Array of constituents managed by CCPP Framework units = none - state_variable = true dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) type = real | kind = kind_phys [ vars_layer_tend ] diff --git a/src/ccpp_hash_table.F90 b/capgen/src/ccpp_hash_table.F90 similarity index 100% rename from src/ccpp_hash_table.F90 rename to capgen/src/ccpp_hash_table.F90 diff --git a/src/ccpp_hashable.F90 b/capgen/src/ccpp_hashable.F90 similarity index 100% rename from src/ccpp_hashable.F90 rename to capgen/src/ccpp_hashable.F90 diff --git a/src/ccpp_scheme_utils.F90 b/capgen/src/ccpp_scheme_utils.F90 similarity index 100% rename from src/ccpp_scheme_utils.F90 rename to capgen/src/ccpp_scheme_utils.F90 diff --git a/ccpp_constituent_prop_mod.F90.patch b/ccpp_constituent_prop_mod.F90.patch new file mode 100644 index 00000000..b1889cac --- /dev/null +++ b/ccpp_constituent_prop_mod.F90.patch @@ -0,0 +1,47 @@ +--- capgen/src/ccpp_constituent_prop_mod.F90 ++++ capgen/src/ccpp_constituent_prop_mod.F90 +@@ -1392,6 +1392,17 @@ + type(ccpp_constituent_properties_t), pointer :: cprop + character(len=dimname_len) :: dimname + character(len=*), parameter :: subname = 'ccp_model_const_table_lock' ++ ! === ONE-OFF cam4 constituent-reorder experiment === ++ ! When .true., force the cam4 advected water species into original-capgen ++ ! order [cloud_liquid=1, cloud_ice=2, water_vapor=3] instead of hash-table ++ ! order, to prove the FWAUT b4b diff is driven purely by constituent order. ++ ! Only the 3 cam4 water-species std-names are remapped; everything else keeps ++ ! its normal hash-order index, so other suites are unaffected unless they ++ ! advect exactly these names. Flip to .false. (or delete) to restore. ++ logical, parameter :: l_const_reorder = .true. ++ integer :: const_pos ++ character(len=512) :: sname_reorder ++ ! === end experiment === + + astat = 0 + errcode_local = 0 +@@ -1460,9 +1471,24 @@ + errcode_local = errcode_local + 1 + exit + end if +- call cprop%set_const_index(index_advect, & ++ ! === ONE-OFF cam4 constituent-reorder experiment === ++ const_pos = index_advect ++ if (l_const_reorder) then ++ call cprop%standard_name(sname_reorder, & ++ errcode=errcode, errmsg=errmsg) ++ select case (trim(sname_reorder)) ++ case ('cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water') ++ const_pos = 1 ++ case ('cloud_ice_mixing_ratio_wrt_moist_air_and_condensed_water') ++ const_pos = 2 ++ case ('water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water') ++ const_pos = 3 ++ end select ++ end if ++ call cprop%set_const_index(const_pos, & + errcode=errcode, errmsg=errmsg) +- call this%const_metadata(index_advect)%set(cprop) ++ call this%const_metadata(const_pos)%set(cprop) ++ ! === end experiment === + else + index_const = index_const + 1 + if (index_const > num_vars) then diff --git a/cmake/ccpp_capgen.cmake b/cmake/ccpp_capgen.cmake deleted file mode 100644 index 599a28ff..00000000 --- a/cmake/ccpp_capgen.cmake +++ /dev/null @@ -1,137 +0,0 @@ -# CMake wrapper for ccpp_capgen.py -# Currently meant to be a CMake API needed for generating caps for regression tests. -# -# CAPGEN_EXPECT_THROW_ERROR - ON/OFF (Default: OFF) - Scans ccpp_capgen.py log for error string and errors if not found. -# HOST_NAME - String name of host -# OUTPUT_ROOT - String path to put generated caps -# VERBOSITY - Number of --verbose flags to pass to capgen -# HOSTFILES - CMake list of host metadata filenames -# SCHEMEFILES - CMake list of scheme metadata files -# SUITES - CMake list of suite xml files -function(ccpp_capgen) - set(optionalArgs CAPGEN_EXPECT_THROW_ERROR) - set(oneValueArgs HOST_NAME OUTPUT_ROOT VERBOSITY KIND_SPECS) - set(multi_value_keywords HOSTFILES SCHEMEFILES SUITES) - - cmake_parse_arguments(arg "${optionalArgs}" "${oneValueArgs}" "${multi_value_keywords}" ${ARGN}) - - # Error if script file not found. - set(CCPP_CAPGEN_CMD_LIST "${CMAKE_SOURCE_DIR}/scripts/ccpp_capgen.py") - if(NOT EXISTS ${CCPP_CAPGEN_CMD_LIST}) - message(FATAL_ERROR "function(ccpp_capgen): Could not find ccpp_capgen.py. Looked for ${CCPP_CAPGEN_CMD_LIST}.") - endif() - - # Interpret parsed arguments - if(DEFINED arg_HOSTFILES) - list(JOIN arg_HOSTFILES "," HOSTFILES_SEPARATED) - list(APPEND CCPP_CAPGEN_CMD_LIST "--host-files" "${HOSTFILES_SEPARATED}") - endif() - if(DEFINED arg_SCHEMEFILES) - list(JOIN arg_SCHEMEFILES "," SCHEMEFILES_SEPARATED) - list(APPEND CCPP_CAPGEN_CMD_LIST "--scheme-files" "${SCHEMEFILES_SEPARATED}") - endif() - if(DEFINED arg_SUITES) - list(JOIN arg_SUITES "," SUITES_SEPARATED) - list(APPEND CCPP_CAPGEN_CMD_LIST "--suites" "${SUITES_SEPARATED}") - endif() - if(DEFINED arg_HOST_NAME) - list(APPEND CCPP_CAPGEN_CMD_LIST "--host-name" "${arg_HOST_NAME}") - endif() - if(DEFINED arg_OUTPUT_ROOT) - message(STATUS "Creating output directory: ${arg_OUTPUT_ROOT}") - file(MAKE_DIRECTORY "${arg_OUTPUT_ROOT}") - list(APPEND CCPP_CAPGEN_CMD_LIST "--output-root" "${arg_OUTPUT_ROOT}") - endif() - if(DEFINED arg_VERBOSITY) - string(REPEAT "--verbose " ${arg_VERBOSITY} VERBOSE_PARAMS_SEPARATED) - separate_arguments(VERBOSE_PARAMS UNIX_COMMAND "${VERBOSE_PARAMS_SEPARATED}") - list(APPEND CCPP_CAPGEN_CMD_LIST ${VERBOSE_PARAMS}) - endif() - if(DEFINED arg_KIND_SPECS) - string(REPLACE "," ";" KIND_SPEC_LIST "${arg_KIND_SPECS}") - set(KIND_ARGS "") # start empty - foreach(pair IN LISTS KIND_SPEC_LIST) - # Append each pair prefixed with --kind-type and quoted. - # The surrounding double‑quotes are added explicitly so the - # resulting string contains them. - set(KIND_ARGS "${KIND_ARGS}--kind-type \"${pair}\"") - string(STRIP "${KIND_ARGS}" KIND_ARGS) - endforeach() - - list(APPEND CCPP_CAPGEN_CMD_LIST ${KIND_SPEC_PARAMS}) - endif() - - message(STATUS "Running ccpp_capgen.py from ${CMAKE_CURRENT_SOURCE_DIR}") - - unset(CAPGEN_OUT) # Unset CAPGEN_OUT to prevent incorrect output on subsequent ccpp_capgen(...) calls. - execute_process(COMMAND ${CCPP_CAPGEN_CMD_LIST} - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - OUTPUT_VARIABLE CAPGEN_OUT - ERROR_VARIABLE CAPGEN_OUT - RESULT_VARIABLE RES - COMMAND_ECHO STDOUT) - - message(STATUS "ccpp-capgen stdout: ${CAPGEN_OUT}") - - if(arg_CAPGEN_EXPECT_THROW_ERROR) - # Determine if the process succeeded but had an expected string in the process log. - string(FIND "${CAPGEN_OUT}" "Variables of type ccpp_constituent_properties_t only allowed in register phase" ERROR_INDEX) - - if (ERROR_INDEX GREATER -1) - message(STATUS "Capgen build produces expected error message.") - else() - message(FATAL_ERROR "CCPP cap generation did not generate expected error. Expected 'Variables of type constituent_properties_t only allowed in register phase.") - endif() - else() - if(RES EQUAL 0) - message(STATUS "ccpp-capgen completed successfully") - else() - message(FATAL_ERROR "CCPP cap generation FAILED: result = ${RES}") - endif() - endif() -endfunction() - -# CMake wrapper for ccpp_datafile.py -# Currently meant to be a CMake API needed for generating caps for regression tests. -# -# DATATABLE - Path to generated datatable.xml file -# REPORT_NAME - String report name to get list of generated files form capgen (typically --ccpp-files) -function(ccpp_datafile) - set(oneValueArgs DATATABLE REPORT_NAME) - cmake_parse_arguments(arg "" "${oneValueArgs}" "" ${ARGN}) - - set(CCPP_DATAFILE_CMD "${CMAKE_SOURCE_DIR}/scripts/ccpp_datafile.py") - - if(NOT EXISTS ${CCPP_DATAFILE_CMD}) - message(FATAL_ERROR "function(ccpp_datafile): Could not find ccpp_datafile.py. Looked for ${CCPP_DATAFILE_CMD}.") - endif() - - if(NOT DEFINED arg_REPORT_NAME) - message(FATAL_ERROR "function(ccpp_datafile): REPORT_NAME not set. Must specify the report to generate to run cpp_datafile.py") - endif() - list(APPEND CCPP_DATAFILE_CMD "${arg_REPORT_NAME}") - - if(NOT DEFINED arg_DATATABLE) - message(FATAL_ERROR "function(ccpp_datafile): DATATABLE not set. A datatable file must be configured to call ccpp_datafile.") - endif() - list(APPEND CCPP_DATAFILE_CMD "${arg_DATATABLE}") - - message(STATUS "Running ccpp_datafile from ${CMAKE_CURRENT_SOURCE_DIR}") - - unset(CCPP_CAPS) # Unset CCPP_CAPS to prevent incorrect output on subsequent ccpp_datafile(...) calls. - execute_process(COMMAND ${CCPP_DATAFILE_CMD} - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - OUTPUT_VARIABLE CCPP_CAPS - RESULT_VARIABLE RES - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_STRIP_TRAILING_WHITESPACE - COMMAND_ECHO STDOUT) - message(STATUS "CCPP_CAPS = ${CCPP_CAPS}") - if(RES EQUAL 0) - message(STATUS "CCPP cap files retrieved") - else() - message(FATAL_ERROR "CCPP cap file retrieval FAILED: result = ${RES}") - endif() - string(REPLACE "," ";" CCPP_CAPS_LIST ${CCPP_CAPS}) # Convert "," separated list from python back to ";" separated list for CMake. - set(CCPP_CAPS_LIST "${CCPP_CAPS_LIST}" PARENT_SCOPE) -endfunction() diff --git a/doc/HelloWorld/hello_scheme.meta b/doc/HelloWorld/hello_scheme.meta index a646c4cd..5c444adb 100644 --- a/doc/HelloWorld/hello_scheme.meta +++ b/doc/HelloWorld/hello_scheme.meta @@ -5,7 +5,7 @@ name = hello_scheme_run type = scheme [ ncol ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension type = integer units = count dimensions = () @@ -33,14 +33,14 @@ [ temp_level ] standard_name = potential_temperature_at_interface units = K - dimensions = (ccpp_constant_one:horizontal_loop_extent, vertical_interface_dimension) + dimensions = (ccpp_constant_one:horizontal_dimension, vertical_interface_dimension) type = real kind = kind_phys intent = inout [ temp_layer ] standard_name = potential_temperature units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = out diff --git a/doc/HelloWorld/temp_adjust.meta b/doc/HelloWorld/temp_adjust.meta index 0f94ed01..15587e4d 100644 --- a/doc/HelloWorld/temp_adjust.meta +++ b/doc/HelloWorld/temp_adjust.meta @@ -5,7 +5,7 @@ name = temp_adjust_run type = scheme [ nbox ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension type = integer units = count dimensions = () @@ -19,7 +19,7 @@ [ temp_layer ] standard_name = potential_temperature units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = inout diff --git a/doc/auto_clone_constituents.md b/doc/auto_clone_constituents.md new file mode 100644 index 00000000..97db84c9 --- /dev/null +++ b/doc/auto_clone_constituents.md @@ -0,0 +1,150 @@ +# `--legacy-auto-clone-constituents` — transient shim + +A capgen CLI flag that re-enables the **auto-clone-static-constituent** +registration path the original ccpp-capgen toolchain provided to +CAM-SIMA. Off by default; turned on with a single flag and a loud +startup banner. Intended as a migration aid — every line of code it +touches is tagged so the whole feature can be removed cleanly when +legacy hosts have moved on. + +## How to enable it + +Pass the flag to both `capgen` and `ccpp_validator` (the +ccpp-physics build system already does this for +`end-to-end-tests/advection_auto_clone/`): + +``` +ccpp_capgen.py --legacy-auto-clone-constituents ... +ccpp_validator.py --legacy-auto-clone-constituents ... +``` + +It is **single-instance only**. If the host metadata declares the +`instance_number` + `number_of_instances` pair (capgen's +multi-instance opt-in), the run aborts with a clear error before +parsing any suite. Legacy hosts predate multi-instance support, so +this restriction matches the use case. + +## What the flag does + +Same shape as original capgen's auto-clone: + +For every scheme argument flagged `advected = True`, `constituent = True`, +or `molar_mass = `, capgen synthesises a `%instantiate(...)` +call into the generated host code, lifting field values straight from +the scheme metadata. The constituent ends up registered in the +per-suite dynamic-constituents buffer alongside any constituents the +host or an explicit register-phase scheme registered. The scheme +author writes no Fortran registration code. + +These metadata attributes are accepted on scheme arguments when the +flag is on (and rejected when it is off): + +| Attribute | Type | Passed to `%instantiate` as | +|----------------------|---------------------|-----------------------------| +| `default_value` | real (kind_phys) | `default_value` | +| `min_value` | real (kind_phys) | `min_value` | +| `water_species` | logical | `water_species` | +| `mixing_ratio_type` | character | `mixing_ratio_type` | + +Fortran-style literals (`0.0_kind_phys`, `1.0d-5`, `-3.14_8`, +`1.0d-5_kind_phys`) are accepted for the real-valued attrs, since +legacy metadata writes the values in source form. + +The other `%instantiate` kwargs (`std_name`, `long_name`, +`diag_name`, `units`, `vertical_dim`, `advected`, `molar_mass`) +already had accepted spellings in capgen; the shim just wires them +into the synthesised call. + +## Defaults that match original capgen + +- **`long_name` is synthesised when missing.** If the scheme metadata + has no `long_name` on a constituent arg, capgen builds one from + the standard name by replacing underscores with spaces and + capitalising the first character. + Example: `cloud_liquid_dry_mixing_ratio` → + `'Cloud liquid dry mixing ratio'`. +- **`diag_name` falls back to the metadata local name** when neither + `diagnostic_name` nor `diagnostic_name_fixed` is set. +- **`vertical_dim` is lifted from the scheme arg's `dimensions = (..., )`** — + whichever entry matches `vertical_layer_dimension` or + `vertical_interface_dimension`; otherwise + `vertical_layer_dimension`. + +## What's stricter than original capgen + +Capgen's general rules apply even with the flag on. Two of them +trip up legacy fixtures: + +1. **Metadata args must match the Fortran subroutine signature.** + Original capgen tolerated a metadata arg-table that listed a + constituent in `_init` even when the Fortran `_init` + didn't accept it as a dummy. Capgen passes the metadata args + at the call site as Fortran keyword arguments, and the validator + catches divergence. Either include the constituent as a Fortran + dummy, or remove it from the init's arg-table. +2. **Base constituents can't be `intent = out`.** A scheme arg with + `advected = True` (or `constituent = True` / `molar_mass = ...`) + on a non-`tendency_of_*` standard name has to be `intent = in` or + `intent = inout`. Only tendency args (std_name starts with + `tendency_of_`) can be `intent = out`. If the legacy scheme + wrote to the array in its init routine, change both the metadata + and the Fortran subroutine to `intent = inout` — the body is + unchanged. + +Multi-instance support is also off-limits while the flag is on (see +above). + +## Used in production CAM-SIMA + +Most CAM-SIMA atmospheric physics schemes rely on the auto-clone +path the same way original capgen ships it: a handful of schemes +register constituents explicitly (`rrtmgp_constituents`, +`musica_ccpp`, `prescribed_aerosols`, `prescribed_ozone`), and the +~16 others (`kessler`, `zm_convr`, `dadadj`, `holtslag_boville_diff`, +`state_converters`, `geopotential_temp`, `cloud_particle_sedimentation`, +…) declare `advected = True intent = inout` on their `_run` +arguments and let the framework register the constituent. Without +the flag, capgen's runtime check fires for every consumer +because no source actually registered the species. The flag closes +that gap by re-creating the auto-clone behaviour from the metadata. + +Production CAM-SIMA does not use `default_value`, `min_value`, +`water_species`, or `mixing_ratio_type` in metadata; the four +extra parser attributes exist to support the advection test (and +any future legacy host that needs the broader kwarg surface on +`%instantiate`). + +## `end-to-end-tests/advection_auto_clone/` — what changed + +The fixture is a port of CAM-SIMA's `ccpp_framework/test/advection_test`. +It exercises the full legacy attr surface (`default_value`, +`diagnostic_name`, `advected`) and the unusual init-phase +`intent = out`-on-base-constituent pattern. Three small edits were +needed to make the port build under capgen: + +1. **`cld_liq.meta`** — in `cld_liq_init`'s `[ cld_liq_array ]` + block, change `intent = out` to `intent = inout`. +2. **`cld_liq.F90`** — in `cld_liq_init`, change + `real(kind=kind_phys), intent(out) :: cld_liq_array(:, :)` to + `intent(inout)`. The body still does + `cld_liq_array = 0.0_kind_phys`; inout is a strict superset. +3. **`cld_ice.meta`** — delete the `[ cld_ice_array ]` block inside + `cld_ice_init`. Fortran `cld_ice_init` only takes + `(tfreeze, errmsg, errflg)`; the run phase already triggers + auto-clone registration of `cloud_ice_dry_mixing_ratio`. + +No `long_name` additions to the metadata were necessary — capgen +synthesises the long_name from the standard name automatically +(see the defaults section above). + +The CTest target `test_advection_auto_clone` passes after these edits. + +## When to retire the flag + +When all consumers have been moved to capgen's explicit +registration model — either by declaring constituents in the host's +`host_constituents(:)` array, or by writing a register-phase scheme +with a `ccpp_constituent_properties_t(:), intent=out` argument that +calls `%instantiate` directly. At that point the flag is no longer +needed by any host, and the shim can be removed in one cleanup +pass. diff --git a/doc/briefing.md b/doc/briefing.md new file mode 100644 index 00000000..34c47057 --- /dev/null +++ b/doc/briefing.md @@ -0,0 +1,475 @@ +# capgen — Briefing for CCPP Framework Developers & Power Users + +*Prepared for the 2026-05-14 walk-through; last revised 2026-06-05. +Companion document to `doc/migration.md` (the detailed migration +guide) and `doc/redesign_prompt.md` (the implementation spec).* + +--- + +## 1. Why a new generator? + +The CCPP Framework runs two code generators today: + +- **`ccpp-prebuild`** — simple, procedural Python; fast; DDT-argument + passing; in production use by NOAA UFS Weather Model, Navy NEPTUNE, + and CCPP-SCM. Reliable but feature-light. No framework-owned + variables. +- **`ccpp-capgen`** — complex, deeply object-oriented Python; flat-field + argument passing; in use by NCAR CAM-SIMA. Many advanced features + designed but never implemented; at UFS/NEPTUNE scale (1200+ variables) + flat-field passing prevents the use of strict error-checking flags + (`-check all`, `-fcheck=all`) required for operational implementation, + and even when it does compile it produces unmaintainably large source + files; nobody on the team fully understands it. + +**`capgen`** starts fresh, drawing lessons from both. Guiding +principle: **simplicity of prebuild, feature set of capgen**. + +What we wanted to fix: + +1. **Flat fields → DDT arguments at all scales.** No "flat-field + group cap" failure mode. +2. **No scope-chain variable promotion.** Variables flow through + metadata, not through a runtime synthetic dictionary stacking. +3. **Code anyone can read and extend.** No 10-deep class hierarchy. +4. **One generator, one CLI, one query tool** for both prebuild-style + and capgen-style hosts. + +--- + +## 2. What capgen is (in one paragraph) + +capgen reads metadata for the **host model**, the **physics +schemes**, and the **suite definition files** (SDFs), produces a +small set of Fortran cap modules that bridge them, and writes a +`datatable.xml` describing the result for CMake / Make to consume. +At runtime the host calls a small set of public entry points +(`ccpp_register`, `ccpp_init`, `ccpp_physics_init`, +`ccpp_physics_run`, `ccpp_physics_*_init`/`_final`, `ccpp_final`); the +generated caps dispatch by `suite_name` (and optionally `group_name`) +to the right scheme. + +--- + +## 3. Core concepts + +### 3.1 Five metadata table types + +| `type = ` | Owner | How it reaches the cap | +|-------------|------------------|----------------------------| +| `scheme` | Physics scheme | Intent args on scheme subs | +| `host` | Host model | Module USE (direct / DDT) | +| `control` | Framework runtime layer | `ccpp_physics_*` args | +| `ddt` | Type definition | Structural — fields only | +| `suite` | Generated suite cap | Module USE | + +### 3.2 Three layers of generated cap + +- **Static API** (`_ccpp_cap.F90`) — public entry points; one per + host build (filename and module name derived from `--host-name`). + Dispatches by `suite_name` → suite cap. +- **Suite cap** (`ccpp__cap.F90`) — per-suite state machine, plus + dispatch by `group_name` → group cap. Suite-owned interstitial data + lives in a sibling `ccpp__data.F90`. +- **Group cap** (`ccpp___cap.F90`) — scheme call sites + with full argument lists, unit/kind/vertical-flip transforms, + optional-arg pointer wrappers, subcycle `do` loops. + +### 3.3 Two-level integer state machine + +Replaces both the boolean `initialized(:)` array from prebuild and +the string-based `ccpp_suite_state` from CAM-SIMA capgen. Per +instance: + +- **Suite-level**: `UNREGISTERED → REGISTERED → FRAMEWORK_INITIALIZED`. +- **Group-level**: `UNINITIALIZED → INITIALIZED → IN_TIMESTEP`. + +### 3.4 Six scheme phases + +`register`, `init`, `timestep_init`, `run`, `timestep_final`, `final`. +`register` is new — schemes that contribute to the constituent table +do so here. `final` replaces the older `finalize` (breaking change, +intentional). + +### 3.5 Variable resolution + +For each scheme arg: + +1. Found in host+control metadata → use the access path. If units / + kind / vertical orientation differ, generate a transform. +2. Not found, first use is `intent(out)` → **suite-owned** variable + (interstitial); add to `ccpp__data.F90`. +3. Not found, first use is `intent(in/inout)` → **error**. +4. Found in suite data (a prior scheme provided it) → use suite data + access path. + +### 3.6 Two tools, one parser + +- `ccpp_capgen.py` — the code generator. Trusts metadata; no + Fortran parsing. +- `ccpp_validator.py` — the standalone Fortran-vs-metadata checker. + The ONE place capgen parses Fortran. Run by developers / + CMake before generation. Checks per-arg `intent`, `type`, `kind`, + and dimension rank; character length must match exactly + (`len=*`↔`len=*`, `len=N`↔`len=N`, no wildcard; old-style F77 + `character*N` / `character*(*)` parsed); DDT and + `external::` types compare against the Fortran + `type(name)` wrapper. `optional` is asymmetric: metadata + `optional=True` against a Fortran-required dummy is an error; the + reverse is a warning. See `doc/migration.md` §7 for the full + rule table. + +Both share the same metadata-parsing library (`metadata/`). + +--- + +## 4. How capgen differs from `ccpp-prebuild` + +| Topic | prebuild | capgen | +|-----------------------------|-----------------------------------|---------------------------------------------------| +| Host metadata mechanism | Hard-coded Python dict (`TYPEDEFS_NEW_METADATA`) | Regular `type = ddt` + `type = host` tables | +| Framework-owned variables | Not supported | First-class (suite-owned interstitial via Case 2) | +| Constituents | Hand-rolled, host-specific glue | Standardized opt-in mechanism with auto-provision | +| `register` phase | Doesn't exist | First phase; schemes declare dynamic constituents | +| Multi-instance API | Implicit, ad-hoc | Paired-opt-in (`instance_number` / `number_of_instances`) | +| Subcycle loop counter | Host plumbs it manually | Registered std names `ccpp_loop_counter` / `ccpp_loop_extent` resolve to the do-loop locals automatically inside `` | +| Suite introspection | Limited | Five runtime queries (`ccpp_physics_suite_list`, `_part_list`, `_schemes`, `_variables`, `_host_data`) | + +--- + +## 5. How capgen differs from `ccpp-capgen` + +| Topic | capgen | capgen | +|-----------------------------|-----------------------------------|---------------------------------------------------| +| Group-cap arguments | Flat fields (1200+ at UFS scale) | DDT arguments (as in prebuild) | +| Variable matching algorithm | Five-layer scope-chain promotion | Flat host+control dict + suite-owned discovery (inherited from prebuild — primary reason runtime is comparable to prebuild) | +| External types (MPI f08 comm, ESMF clock) | Tabled (solution complexity) | First-class via `type = external::` | +| `type = module` in metadata | Yes | Renamed `type = host` | +| `is_constituent` scheme args | Auto-cloned by generator | Schemes register constituents explicitly in the `register` phase via `ccpp_constituent_properties_t(:)`; original capgen's auto-clone path is available behind the opt-in `--legacy-auto-clone-constituents` shim for legacy hosts (single-instance only) | +| `ConstituentVarDict` | Synthetic scope between suite + host | Removed; constituents are one of four sources (`control`/`host`/`suite`/`constituent`) on `ResolvedArg` | +| `_state` runtime check | String | Integer (named parameters) | +| Fortran-vs-metadata check | Inside the generator | Separate tool (`ccpp_validator.py`) | +| Code complexity | Deep OO hierarchy | Flat data classes + procedural resolver | + +--- + +## 6. Breaking metadata changes hosts must make + +Comprehensive list — see `doc/migration.md` for full detail. + +### 6.1 Table types + +- `type = module` → **`type = host`**. + +### 6.2 Phase names + +- `_finalize` → **`_final`** in both metadata and + Fortran source. + +### 6.3 Standard names + +- `horizontal_loop_extent` → **`horizontal_dimension`** uniformly in + scheme metadata. (The chunk-vs-full-domain distinction is driven + by what the host passes for `horizontal_loop_begin` / + `horizontal_loop_end`.) +- `number_of_openmp_threads` → **`number_of_threads`** (matches the + `thread_number` control variable convention). + +Both are rewritten on the fly by **`--legacy-mode`** for a transition +period; the shim prints a banner listing every rewrite it performs +and is marked for clean removal. + +### 6.3b Transient migration shims — full set + +Three opt-in CLI flags exist for migrating legacy hosts. Each lives +in its own self-contained module, prints a loud banner at startup, and +every touchpoint is grep-tagged for clean removal: + +| Flag | What it does | Removal grep | +|---------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------| +| `--legacy-mode` | Parse-time substitution of two deprecated CCPP standard names (see §6.3 above). Active on both `ccpp_capgen.py` and `ccpp_validator.py`. | `legacy-compat` | +| `--gfs-dim-aliases` | Treats GFS-physics names `adjusted_vertical_layer_dimension_for_radiation` and `vertical_composition_dimension` as equivalent to `vertical_layer_dimension` **inside the upper-bound dim identity check only** (the variables themselves stay distinct). Resolver-only, so `ccpp_capgen.py` carries the flag; `ccpp_validator.py` does not (the validator never reaches the dim canonicaliser). | `dim-aliases` | +| `--legacy-auto-clone-constituents` | Reinstates original ccpp-capgen's auto-clone-static-constituent registration path: every `is_constituent` consumer (`advected = True` / `constituent = True` / `molar_mass = …`) with no register-phase source is auto-registered into the per-suite dynamic-constituents buffer using values lifted straight from the scheme metadata. Adds four legacy `%instantiate` kwargs to the parser (`default_value`, `min_value`, `water_species`, `mixing_ratio_type`). **Single-instance only** — declaring the `instance_number` + `number_of_instances` pair while the flag is on is a hard error. Available on both `ccpp_capgen.py` and `ccpp_validator.py`. | `auto-clone-constituents` | + +All three flags are listed as runways, not destinations: drop the +underlying legacy spelling from host/scheme metadata and the flag can +be retired. See `doc/auto_clone_constituents.md` for the full +auto-clone reference. + +### 6.4 Required host `type = control` table + +Every host MUST declare scalar integers (and one character) with +these CCPP standard names: + +- `suite_name`, `horizontal_loop_begin`, `horizontal_loop_end`, + `number_of_physics_threads`, `ccpp_error_code`, `ccpp_error_message`. + +**Two paired-optional control pairs** — declare *both* members of a +pair (in `type = control`) or *neither*; declaring exactly one is a +hard error: + +- `instance_number` + `number_of_instances` → opt into the + multi-instance API. +- `thread_number` + `number_of_threads` → opt into the multi-threading + API. + +The two pairs are fully symmetric. Declaring a pair makes the index a +per-call control argument; omitting it drops both args and the +framework uses literal `1` where the index would go. A host variable +may be dimensioned by `number_of_instances` / `number_of_threads` only +when its pair is declared (otherwise the scalar-index collapse can't +find the index variable and errors). + +### 6.5 DDT-instance variables with scalar-index dims + +Container DDT-instance variables (`physics%Interstitial`, +`physics%Coupling`, ...) dimensioned by a count standard name +(`number_of_threads`, `number_of_instances`) get their scalar index +inserted **automatically** by capgen. The host metadata declares +the dim; the generator emits +`physics%Interstitial(thread_number)%alpha(...)` at every call site. + +The host's Fortran can keep its existing OpenMP-thread-private DDT +layout — no glue code needed on the host side. + +### 6.6 Leaf variables MUST NOT carry registered scalar-index dims + +Rule 2 of the registered-scalar-index-dimension contract: scalar +variables (real / integer / character / DDT-typed leaves the scheme +binds to) cannot declare `number_of_threads` or `number_of_instances` +as a dimension. Wrap them in a container DDT instead. This is +enforced at parse time with an explicit remediation message; existing +CCPP-physics, UFS-WM, and CAM-SIMA host metadata is already +compliant. + +### 6.7 No more `cdata` / `ccpp_t` struct passing + +The framework-owned bag-of-state struct is replaced by explicit +control-variable arguments to the public entry points. + +--- + +## 7. What capgen does NOT support (yet) + +### 7.1 Deferred — to be resolved in upcoming work + +- **Constituents overhaul.** Three reform proposals on the table + (`doc/constituents_overhaul.md`); decision pending an upcoming + meeting. Pieces involved: framework setter additions + (`set_advected`, `set_diagnostic_name`, `set_default_value`), + `is_match` relaxation, Class A vs Class B property classification. +- ~~**Validator host-metadata check.**~~ **Landed 2026-06-01**: + `ccpp_validator.py --host-files` validates `type = host` and + `type = ddt` tables against the Fortran (`doc/migration.md` §7.4). +- **Codegen-time scheme-registration cross-check.** Today's + registration check is at runtime + (`ccpp_initialize_constituents`). Stronger options: new metadata + attribute `registers_std_names = a, b, c` on register-phase + tables; cross-check at codegen. +- **Nested-subcycle `ccpp_loop_counter` semantics.** When a scheme + inside a deeply nested subcycle asks for `ccpp_loop_counter`, it + currently resolves to the **outermost** loop's counter. None of + the in-tree physics catalogs uses the inner-counter case. +- **`ccpp_datafile.py --host-files` repurpose.** The current + `--host-files` returns the generated host-API file; should be a + filtered list of *input* host metadata files (parallel to the new + `--scheme-files`). Deferred. +- **`ccpp_host_constituents.F90` suppression** when no suite touches + constituents (file is correct-but-empty under host-wins; should + not be emitted at all). +- **Python linter / formatter pass.** Pick `ruff`, apply across + `capgen/`. + +### 7.2 Intentionally NOT supported + +- **`_finalize` phase spelling.** Use `_final`. No legacy-mode + shim — rename in metadata + Fortran. +- **`type = module`.** Use `type = host`. +- **Flat-field scheme call arguments** (capgen's failure mode). +- **`character(len=*)` as a DDT component** (Fortran disallows it; + we error at parse time with a remediation pointing at + `character(len=:)` deferred-length). +- **Multiple registration sources for the same constituent** with + silent dedup. Today's behavior is to error on conflict; the + proposed reform sets a clear precedence rule (host-set Class B + properties win) — pending the constituents-overhaul decision. +- **`ConstituentVarDict`** synthetic scope between suite and host. + Gone for good. + +--- + +## 8. Validation and error reporting + +A deliberate design choice across capgen: **errors are loud, +specific, and actionable**. Examples surfaced during the SCM +shake-down: + +- Empty `units =` line → error names file, line, variable, + attribute, raw value, AND inner reason. +- Scheme metadata file passed via `--scheme-files` but missing from + the SDF → silently ignored (and dropped from `` so + CMake doesn't compile orphan code). +- Scheme listed in the SDF but its metadata not supplied → single + CCPPError listing every missing scheme + pointer to + `--scheme-files`. Replaces silent empty-cap emission. +- DDT-instance variable with a non-registered scalar-index dim AND + flattenable fields → error shows the broken access pattern + capgen WOULD have emitted and quotes the Fortran compiler + error verbatim ("Component to the right of a part reference with + nonzero rank must not have the POINTER attribute"). +- Generated `case default` on `select case(suite_name)` / + `select case(group_name)` → unknown suite or group at runtime + produces a clear errflg + errmsg, not silent fall-through. + +--- + +## 9. Build-system integration (capsule view) + +```cmake +# In your CMakeLists.txt +set(SCHEME_METADATA_FILES …list of .meta paths…) +set(HOST_METADATA_FILES …list of host .meta paths…) +set(SUITE_FILES …list of suite XML paths…) + +# Validate before generation (developer step, optional in CI) +ccpp_validator(SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES}) + +# Run the code generator +ccpp_capgen(HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT ${OUTPUT_ROOT}) + +# Pull the manifest from the datatable +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES ${CCPP_FILES}) + +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) + +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +add_library(scm-ccpp STATIC + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES}) +``` + +Regenerating on every CMake configure is cheap — `write_if_changed` +preserves mtimes when content hasn't changed, so `make` / `ninja` +don't rebuild downstream objects unless something actually moved. + +--- + +## 10. Where things stand right now + +- **Unit tests**: 1516 passing on `feature/capgen` (as of + 2026-06-05). +- **End-to-end tests passing** (12): `advection`, + `advection_auto_clone`, `capgen`, `chunked_data`, + `constituents_dim`, `ddthost`, `instances`, `instances_advection`, + `nested_suite`, `opt_arg`, `suite_allocate`, `var_compat`. The two + newest — `constituents_dim` (a variable dimensioned by + `number_of_ccpp_constituents`) and `suite_allocate` (suite-owned + allocatable interstitials sized by a scheme-written dimension) — were + added while hardening the CAM-SIMA HPC build. +- **Code size**: ~17.8k LOC of Python under `capgen/` (includes + docstrings, inline comments, and the three transient shim modules) + + ~18k LOC of unit/doctest under `unit-tests/`. Still procedural, + still flat data classes. +- **Three transient migration shims now live** (see §6.3b): + `--legacy-mode` (2026-05-13), `--gfs-dim-aliases` (2026-05-21), + and `--legacy-auto-clone-constituents` (2026-05-21). Each is + isolated in its own module + grep-tag so removal is a single + cleanup pass once the underlying legacy spelling is gone from + host/scheme metadata. +- **CCPP-SCM**: actively driving development — every build / runtime + failure surfaced this month landed as a fix in capgen (rather + than being patched around in the host). Most of the `phys_ps` group + now builds end-to-end via `--legacy-mode` + `--gfs-dim-aliases`. + On 2026-05-20 the per-arg-attribute validator caught **67 real + metadata/Fortran disagreements** in the SCM physics tree (12 missing + `kind = kind_phys` + 42 intent mismatches + a mix of optional-flag + and bare-`real` cases); all fixed. +- **Validator** now checks per-argument `intent`, `type`, `kind`, and + dimension rank in addition to the original name/count check. + Asymmetric `optional` rule, DDT + `external::` + type normalisation, exact character-length match (no `len=*` + wildcard; old-style F77 `character*N` parsed). +- **Resolver cross-metadata checks** (late 2026-05-20): host/scheme + (and suite-owned-var first-writer/follow-on) consistency on type, + rank, and per-position dimension entries. Default lower bound has + three equivalent spellings (bare `X`, `1:X`, `ccpp_constant_one:X`); + other lower bounds stay distinct. Numeric kind remains lenient + (transform path). See `doc/migration.md` §1.3.2. +- **Host `active` + scheme arg shape**: when the scheme arg is + optional, the cap uses pointer association (PRESENT()-aware); when + the scheme arg is non-optional, the cap emits a runtime guard + (`if (.not. (active)) errflg = 1; return`) before the call. + Replaces an earlier static rule that forced scheme metadata to lie + about optionality. See `doc/migration.md` §1.3.1. +- **`--no-host-introspection`** (2026-05-14): stubs the bodies of + the five suite-introspection routines in `_ccpp_cap.F90`, + shrinking the file from ~33k lines to ~800 for the 10-suite SCM + build (the introspection case-blocks were making even `-O1` + compilation effectively hang). Signatures stay so existing host + callers still link; stubbed bodies return `errflg = 1` with a clear + `errmsg`. +- **NEPTUNE**: cleanup and acceptance testing in progress. + Regular/lower-atmosphere physics builds and runs and produces + results within tolerance (deviations similar to compiler changes). + High-altitude physics testing is next. +- **UFS Weather Model**: not yet attempted; SCM is the proving + ground first. An anticipated complication is the "fast physics" + called directly from the FV3 dynamical core as a separate group. +- **CAM-SIMA**: **reconnected (2026-06-03 → 06-05).** capgen now + drives the real CAM-SIMA build on Derecho via a thin compatibility + layer (`cime_config/capgen_compat/`, in the CAM-SIMA tree) that + re-implements original ccpp-capgen's Python API surface + (`cap_database`, `host_model_dict`, `call_list`, the per-variable + `Var` accessors) on top of capgen's `datatable.xml` + + `ResolvedArg` / `HostVarEntry`. CAM-SIMA's `cam_autogen.py`, + `generate_registry_data.py`, and `write_init_files.py` are unchanged. + Three cases build **and run to completion** on Derecho under **both + gnu and intel** (bit-comparable results): `kessler`, `rrtmgp`, and + `se_cslam` / CSLAM (the FCAM7 `cam7` suite — the full convection + + stratiform + radiation + gravity-wave physics). A short shareable + brief on the compatibility layer is `doc/capgen_compat_layer.md`. + `--legacy-auto-clone-constituents` is still the no-decision-needed + bridge for the ~16 schemes that rely on auto-clone. Bring-up this + week produced three reusable lessons baked into the docs: the + consume-without-re-flagging rule (`doc/migration.md` §6.5), the + adapter must key constituent handling on `ResolvedArg.source` + (`doc/constituents_overhaul.md` §4.15), and `module_name` overrides + are required wherever a Fortran module name differs from its + `.meta` table name (`doc/migration.md` §3.3). + +--- + +## 11. Walk-through outline (suggested order for the meeting) + +1. Live `ccpp_capgen.py --help` (CLI shape). +2. Show one scheme's `.meta` + its generated group-cap fragment. +3. Run the generator twice — note the `Unchanged: …` messages on the + second pass (write-if-changed in action). +4. Run `ccpp_datafile.py --scheme-files datatable.xml` to show the + filtered manifest. +5. Demonstrate a deliberately-broken metadata (`units =` empty, or + missing scheme, or invalid `case default` group) to show the + error UX. +6. Walk through the registered scalar-index dimension table and the + two rules. +7. Open the floor — focus areas for the audience: + - **Host metadata maintainers**: anything in §6 that surprises + you for your model? + - **Scheme metadata maintainers**: anything in §6.2 / §6.3 that + can't be migrated cleanly? + - **Framework devs**: §7.1 — which deferred items block your + downstream work? diff --git a/doc/briefing_pm.md b/doc/briefing_pm.md new file mode 100644 index 00000000..8a25cfd9 --- /dev/null +++ b/doc/briefing_pm.md @@ -0,0 +1,404 @@ +# capgen — Briefing for Project Management + +*Companion to `doc/briefing.md` (the developer walk-through) and +`doc/redesign_analysis.md` (the deep-dive technical comparison of +prebuild and capgen). This document targets project leadership and +program managers; it summarises the case for `capgen` in terms of +product risk, schedule, and cross-organization impact rather than +implementation detail.* + +*Last revised: 2026-06-05.* + +--- + +## TL;DR + +The CCPP Framework today ships **two** code generators that solve the +same problem differently: + +- **`ccpp-prebuild`** powers NOAA UFS, Navy NEPTUNE, and CCPP-SCM. + Simple and reliable, but feature-light — does not support features + CAM-SIMA needs (constituents, framework-owned variables, + introspection). +- **`ccpp-capgen`** powers NCAR CAM-SIMA. Feature-rich, but built on + technical choices that **do not scale** to UFS or NEPTUNE and that + **do not support multi-instance hosts** at all. + +Neither generator can be the basis for a single shared toolchain. +**`capgen`** is a third generator, started in early May 2026, +designed to do everything both other generators do, in code small +enough for a few people to own, with the architectural choices that +make it work at UFS/NEPTUNE scale and beyond. The redesign is +running on the SCM as proving ground; UFS / NEPTUNE / CAM-SIMA +re-integration is sequenced behind that. + +This document explains, in plain language, **why we did not extend +capgen instead**, what risks the redesign retires, and where things +stand. + +--- + +## 1. The three generators in one paragraph each + +**`ccpp-prebuild`** (NOAA/NAVY/DTC, in production for UFS/NEPTUN/SCM). +Procedural Python; reads metadata; emits Fortran caps; passes +host-defined derived-type (DDT) arguments to scheme call sites. In +production for several years; bug rate is low; the team understands +it (those who worked with it). What it doesn't do: framework-owned +variables, the constituent mechanism CAM-SIMA needs, and runtime +introspection. Treated as the **baseline for simplicity and +reliability**. + +**`ccpp-capgen`** (NCAR, in production for CAM-SIMA). Heavy +object-oriented Python (deep class hierarchy, ~tens of thousands of +lines); reads metadata; emits Fortran caps that pass **flat scalar +fields** to scheme call sites instead of DDTs; supports the +constituent mechanism, suite-owned variables, introspection, and a +few other features prebuild lacks. **It is the only existing +generator with those features.** But — see §3 — it has structural +limits that make it impractical for UFS, NEPTUNE, or multi-instance +hosts, and the implementation is concentrated enough that few people +can extend it safely (if at all - primary developer gone). + +**`capgen`** (new, 2026-05). Procedural Python (~17.8k lines +including inline comments and the three transient shim modules; flat +data classes); reads the same metadata format; passes +arguments like prebuild; supports the features capgen pioneered +(constituents, suite-owned variables, introspection); supports +multi-instance, an integer state machine, six explicit scheme +phases, vertical-flip / unit / kind transforms, registered +scalar-index dimensions for threading and ensembles, write-if-changed +build integration, and a separate Fortran-vs-metadata validator +tool. Designed so the same generator works for prebuild-style hosts +(UFS / NEPTUNE / SCM) and capgen-style hosts (CAM-SIMA). + +--- + +## 2. Why this matters now + +Three pressures converged in 2025/26: + +1. **Framework unification heavily delayed.** A fully-functional + capgen that supports UFS / NEPTUNE / SCM and replaces prebuild + was promised for years, and never delivered. Pressure from + project management and sponsors is building. +2. **UFS / NEPTUNE want the capgen feature set.** Constituents + in particular are increasingly central to atmospheric physics + (chemistry, aerosols, deep atmosphere), and re-implementing the + prebuild-side glue per host is duplicated effort. Extending + capgen to UFS-scale runs into the flat-field problem (§3.1) — + not a small refactor, a fundamental data-shape change. The + performance of capgen generating multi-suite caps is up to + 20 times slower than that of prebuild for the CCPP SCM. This + is caused by fundamental design choices (five layers of + classes inheriting from each other) that are integral to capgen. +3. **The team owning capgen has limited bandwidth to extend it.** + The class hierarchy is intricate; understanding the + `ConstituentVarDict` scope-chain or the auto-clone path requires + reading several modules together. Realistically, only one or two + people on the framework team can change capgen without breaking + something downstream. One of them now lives overseas. This is an + unacceptable **bus-factor risk** that the redesign retires. + +--- + +## 3. What capgen does that does not extend to UFS / NEPTUNE / multi-instance + +This section is for the project lead who came from the capgen side: +none of these are critiques of capgen as a *product*. They are +specific architectural choices that worked for CAM-SIMA's +single-instance design and don't generalize. Each is sourced from +the technical analysis in `doc/redesign_analysis.md` and validated +by the SCM / multi-instance test work this month. + +### 3.1 Flat-field argument passing fails at UFS / NEPTUNE scale + +CAM-SIMA's group caps pass **every individual variable as a separate +argument** to the scheme dispatch routine. At CAM-SIMA's roughly +two-hundred-variable scale this works. At UFS scale (~1200 +variables per group), the generated Fortran exceeds compiler limits, +prevents the use of strict error-checking flags (`-check all`, +`-fcheck=all`) required for operational implementation, and even +when it does compile produces unmaintainably large source files. +**This is one technical reason capgen cannot drive UFS today**, +independent of any other concern. + +`capgen` reverts to prebuild's DDT-argument convention. Host +authors pass their physics DDTs by reference (one or a few arguments +per scheme call); component access happens **at the scheme call level**. +This works at every scale we've measured. + +### 3.2 Single-instance constituents are baked into the generated code + +CAM-SIMA runs one host per executable, so capgen generates a single +module-level `ccpp_model_constituents_obj`. The constituent +mechanism — the central feature capgen inherited from capgen — +references that global directly. Re-targeting capgen to +multi-instance is not a configuration toggle; it requires +re-emitting the constituent module per-instance throughout the +generator, plus refactoring the framework setters. + +`capgen` was multi-instance **from day one**: every constituent +entry point takes an `instance_number` argument; the property +storage, the state machine, the dynamic-constituent buffers are all +per-instance. As of 2026-05-18, the per-suite dynamic-constituents +buffer was also moved per-instance after the new combined +multi-instance + constituents end-to-end test surfaced a latent bug +that capgen would never have hit (because capgen never supported +multi-instance). **The redesign is finding bugs the legacy +toolchain hid.** + +### 3.3 Constituent registration has three competing paths in capgen + +capgen accepts constituent declarations from (a) host-supplied +arrays, (b) scheme `register`-phase Fortran subroutines, and (c) an +**auto-clone path** that scans scheme metadata for the +`is_constituent` attribute and silently generates a registration in +the host cap. The auto-clone path is invisible from the scheme +Fortran — to know whether a scheme registers a constituent you have +to know the generator semantics. This makes scheme code harder to +read, harder to port between hosts, and harder to debug when +registrations collide. + +`capgen` keeps only the first two (explicit) paths. Auto-clone +is deliberately gone from the default behaviour — see +`doc/constituents_overhaul.md` §2.3. For legacy hosts that already +ship metadata in the original-capgen shape (production CAM-SIMA's +atmospheric_physics tree is the immediate consumer; ~16 of the ~20 +schemes that touch constituents rely on auto-clone today), an opt-in +shim `--legacy-auto-clone-constituents` reinstates the original path +behind a single CLI flag with a loud startup banner; see +`doc/auto_clone_constituents.md`. The shim is single-instance only +and is marked for removal once consumers migrate to explicit +registration. + +### 3.4 Host-specific values baked into scheme metadata + +capgen requires `diagnostic_name` (host's diagnostic-output label, +e.g. `CLDLIQ` for CAM-SIMA but something else for UFS) at +constituent instantiation time. Schemes therefore embed +host-specific strings into their own metadata. Porting a scheme +between hosts requires either editing the scheme or maintaining a +fork. + +`capgen` is moving `diagnostic_name` (and a handful of other +host-configuration properties) to a host-side override mechanism; +schemes carry physics-portable defaults only. The reform is +documented in `doc/constituents_overhaul.md`; the decision is on the +agenda for one of the next framework-team meetings. + +### 3.5 Synthetic variable-resolution scopes are hard to extend + +capgen introduces a five-layer deep synthetic dictionary +(`ConstituentVarDict`) between the suite and host scopes during +variable matching. The mechanism works for capgen's use cases +but is a code path most contributors don't read. Extending the +resolver to handle multi-instance dimensions, scalar-index +substitution, or constituent host-wins semantics required +undoing parts of the synthetic scope. + +`capgen`'s resolver is flat: each scheme arg is classified into +exactly one source (control / host / suite / constituent), recorded +on a small data class (`ResolvedArg`), and used directly by the +emitter. No synthetic dictionary. **This design inherits from +`prebuild` and is the primary reason `capgen` is comparable +in performance to `prebuild`. + +### 3.6 Code volume and team coverage + +capgen is roughly an order of magnitude larger than prebuild, with a +deeply layered class hierarchy. This is not a moral failing — it +reflects the feature set — but the practical consequence is that +the maintenance burden falls on a small subset of the framework +team. capgen is comparable to prebuild in *shape* (procedural +Python with small data classes — no deep class hierarchy), and the +generator itself sits at ~17.8k lines. +The "who can fix this" pool is closer to "anyone with +framework context". capgen comes with ~1.4k docstring + unit +tests (~18k lines of test code), plus an end-to-end test suite of +12 fixtures that covers all of prebuild's and capgen's existing +end-to-end tests and adds new ones for multi-instance + constituents +(`instances_advection`), the auto-clone-constituents shim +(`advection_auto_clone`), constituent-count dimensions +(`constituents_dim`), and suite-owned allocatable interstitials +(`suite_allocate`). Including these tests and the rich inline +comments puts capgen's full tree on the same order of magnitude as +capgen — about half of which is test coverage and human-readable +prose, not load-bearing logic. + +--- + +## 4. What `capgen` does better than capgen — at any scale + +For audiences who already accept the multi-instance and UFS-scale +arguments, the day-to-day quality-of-life improvements that apply +even to CAM-SIMA-shape problems: + +| Topic | capgen | capgen | +|---|---|---| +| Scheme call argument shape | Flat fields | DDT references | +| Variable resolution | Scope-chain promotion via synthetic dict | Flat 4-source classification on `ResolvedArg` | +| Suite state runtime check | String comparison | Integer-named-parameter state machine | +| Fortran-vs-metadata validation | Embedded in generator | Standalone tool (`ccpp_validator.py`) — run by developers or CMake before generation | +| Generator code style | Deep class hierarchy | Flat data classes + procedural resolver | +| Error reporting | Variable amount of context | "Loud, specific, actionable" enforced — every parse-time error names file, line, variable, attribute, value, and reason | +| Constituent registration | Three sources (one invisible) | Two sources, both explicit | +| `is_constituent` auto-clone | Yes (host-specific values baked into scheme metadata) | Removed by default; reinstated for legacy hosts behind opt-in `--legacy-auto-clone-constituents` shim (single-instance only) | +| `_finalize` vs `_final` phase name | `_finalize` | `_final` (renamed to keep symmetry with init/timestep_init/timestep_final) | + +--- + +## 5. Additional features of `capgen` compared to `capgen` + +Features that exist only in capgen (some exist in prebuild): + +| Capability | Why it matters | +|---|---| +| **Multi-instance host support** (per-instance state machine, per-instance constituent objects, per-instance dynamic-constituents buffers as of 2026-05-18) | Required by NEPTUNE (prebuild has basic solution) | +| **Registered scalar-index dimensions** | When metadata says a variable is dimensioned by `number_of_threads` or `number_of_instances`, capgen injects the right per-call subscript automatically; the host's OpenMP-thread-private DDT layout works unchanged | +| **Subcycle loop-counter automation** | Schemes inside a `` element can access `ccpp_loop_counter` / `ccpp_loop_extent` directly; the generator emits the Fortran `do` loop and binds the locals | +| **`--legacy-mode` migration shim** | One CLI flag enables silent rewrite of two known-good deprecated standard names (`horizontal_loop_extent` → `horizontal_dimension`, `number_of_openmp_threads` → `number_of_threads`) with a loud warning — buys time for host metadata to migrate | +| **`--gfs-dim-aliases` migration shim** (2026-05-21) | One CLI flag treats GFS-physics names (`adjusted_vertical_layer_dimension_for_radiation`, `vertical_composition_dimension`) as equivalent to `vertical_layer_dimension` in the dim-identity check only — variables remain distinct everywhere else. Resolver-only; clean grep-revert. Required for CCPP-SCM v17p8 to build under capgen. | +| **`--legacy-auto-clone-constituents` migration shim** (2026-05-21) | One CLI flag reinstates original ccpp-capgen's auto-clone-static-constituent registration path for the ~16 production-CAM-SIMA schemes that depend on it. Single-instance only (predates multi-instance); fails fast if a multi-instance host is supplied. This is the no-decision-needed bridge that lets capgen accept CAM-SIMA's atmospheric_physics metadata before any constituent-overhaul work lands. | +| **`--no-host-introspection` flag** | The five runtime introspection routines (`ccpp_physics_suite_list`, etc.) emit large `select case` blocks at SCM scale; this flag stubs the bodies, dropping the generated static API from ~33,000 lines to ~800 for the SCM build (the introspection routines were making `-O1` compilation effectively hang) | +| **Consistent handling of external types** (MPI f08 communicator, ESMF clock) | Tabled in capgen because of the complexity of the solution | + +--- + +## 6. Where things stand right now (2026-06-05) + +- **Unit tests**: 1516 passing. No known failures. +- **End-to-end tests**: 12 passing — `advection`, + `advection_auto_clone` (CAM-SIMA advection_test port exercising the + auto-clone shim), `capgen`, `chunked_data`, `constituents_dim`, + `ddthost`, `instances`, `instances_advection` + (multi-instance + constituents), `nested_suite`, `opt_arg`, + `suite_allocate`, `var_compat`. The two newest (`constituents_dim`, + `suite_allocate`) were added while hardening the CAM-SIMA HPC build. +- **Code size**: ~17.8k lines of Python under `capgen/` including + inline comments and the three transient shim modules; ~18k lines of + unit/doctest under `unit-tests/`. Still procedural; still flat + data classes; still well below capgen. +- **CCPP-SCM**: actively driving development. Each build / runtime + issue surfaced this month landed as a fix in capgen rather than + a host-side workaround. All available suites in CCPP-SCM now + build and run end-to-end via `--legacy-mode` + `--gfs-dim-aliases`. +- **Three transient migration shims in place** (see §5). Each is + isolated in its own module with a single grep tag, so removal once + hosts migrate is a single cleanup pass. +- **Auto-clone shim landed 2026-05-21**. Reinstates original capgen's + auto-clone path behind `--legacy-auto-clone-constituents`. This is + the no-decision-needed bridge for CAM-SIMA — the ~16 schemes that + declare `advected = True` in `_run` arg-tables and rely on the + framework to register the constituent will now work under capgen + without metadata edits. +- **Multi-instance + constituents fix landed 2026-05-18**. The new + combined end-to-end test surfaced a latent shared-buffer mutation + bug; the fix moves the per-suite dynamic-constituents buffer + per-instance. No coordination with CAM-SIMA / UFS / NEPTUNE + required (host-facing API unchanged). +- **NEPTUNE**: Final cleanup and acceptance testing in progress. + All regression tests (~300) pass with the three mandatory + compilers (Intel LLVM, GCC, LLVM native) for regular physics, + mid-altitude, and high-altitude physics (feature-complete). +- **UFS Weather Model**: not yet attempted; SCM is the proving + ground first. Expecting updates due to the "fast physics" + called directly from the FV3 dynamical core as separate group. +- **CAM-SIMA**: **re-connected (2026-06-03 → 06-05).** capgen now + drives the production CAM-SIMA build on the Derecho supercomputer + through a small compatibility layer that lets CAM-SIMA's existing + build scripts call capgen without being rewritten. Three + configurations build **and run to completion under both the Intel and + GNU compilers**, with bit-comparable results: `kessler`, `rrtmgp`, + and `se_cslam`/CSLAM — the last being the full CAM7 physics suite + (deep + shallow convection, stratiform microphysics, RRTMGP + radiation, gravity-wave drag) on a cubed-sphere/CSLAM-advection + configuration. This is the first time the redesigned generator has + produced a complete, running CAM-SIMA model. The constituent + overhaul decision (see §7) remains a separate track and was not on + the critical path for this milestone. + +--- + +## 7. What is intentionally NOT decided yet + +The redesign is opinionated about the architectural choices (DDT +arguments, per-instance everything, integer state machine, two-tool +split). It is **not** opinionated about the framework-level +constituent reform. + +`doc/constituents_overhaul.md` lays out three reform proposals on +the table: + +- **Proposal A** (mostly landed): bug-fix on the deallocate path + + add missing host setters for properties the host wants to + override. Conservative. +- **Proposal B** (recommended for the next 4–6 weeks): relax the + identity-equality check, formally classify properties as + "scheme-intrinsic" (immutable) vs "host-configuration" (mutable + after registration). Physics schemes using constituents become + genuinely portable across hosts. +- **Proposal C** (tabled): drop scheme-side constituent + registration entirely; only the host registers. Cleaner but + requires coordinated PRs across the framework, both generators, + the CAM-SIMA atmospheric_physics tree, and CAM-SIMA itself. + +These are open questions for the framework-team meeting, not +capgen decisions. capgen is structured so all three +proposals are implementable on top of it. + +--- + +## 8. Risk register (project-management view) + +| Risk | Status | Mitigation | +|---|---|---| +| capgen diverges from capgen feature set | LOW | Cross-checked by `doc/redesign_analysis.md`; the feature comparison table in §4 / §5 is exhaustive | +| Host metadata break for UFS / NEPTUNE / CAM-SIMA | LOW | Three transient shims (`--legacy-mode`, `--gfs-dim-aliases`, `--legacy-auto-clone-constituents`) together cover the known-incompatible standard-name pair, the GFS radiation/composition vertical-dim spellings, and original capgen's auto-clone registration path. Remaining required changes (e.g., `_finalize` → `_final`) are mechanical and listed in `doc/migration.md` §3 | +| Constituent overhaul stalls | LOW | Proposal A unblocks the immediate bug; capgen works with the current framework today; `--legacy-auto-clone-constituents` lets CAM-SIMA's atmospheric_physics build without an overhaul decision; the overhaul is a separate decision track | +| Bus-factor on capgen itself | MEDIUM | Procedural code style + flat data classes + 1426-test safety net; significantly lower than capgen's bus factor | +| Two host call-shape conventions (prebuild-style vs capgen-style) coexist forever | LOW | capgen emits one shape; downstream host conversions are tracked in `doc/migration.md` | +| Regression discovered during NEPTUNE / UFS testing | EXPECTED | SCM proving ground catches most; remaining issues become capgen tickets, not host-side patches | +| ccpp-prebuild end-of-life requires a sunset plan | OPEN | Not yet scoped; both generators currently coexist in the framework repo | + +--- + +## 9. The pragmatic case (for the meeting) + +Three points worth raising explicitly: + +1. **Extending capgen to UFS/NEPTUNE scale is not a configuration + change — it is a refactor of the same magnitude as a redesign.** + The flat-field convention is load-bearing throughout capgen's + variable-matching, resolution, and emission code. Once that + change is made, the resulting generator looks substantially + like capgen anyway. +2. **The features capgen pioneered (constituents, suite-owned + variables, introspection) are kept and improved — not + discarded.** capgen is genuinely the successor, not a + parallel project. The contributions made on the capgen side are + what made the capgen feature set possible. A significant + portion of capgen's code, in particular metadata parsing, + Fortran-metadata validation, and constituents, were imported + into capgen. +3. **The team owning capgen can be larger than the team owning + capgen.** This is the most important practical point for + long-term program health. A framework that three organizations + can maintain is more resilient than a framework that one + organization (or one individual in that organization) can maintain. + +--- + +## 10. References + +- `doc/briefing.md` — developer walk-through; same outline, more + technical detail. +- `doc/redesign_analysis.md` — deep-dive technical comparison of + prebuild and capgen with named-product examples. +- `doc/migration.md` — host-author migration guide. +- `doc/constituents_overhaul.md` — the constituent-reform discussion + document. +- `doc/capgen_compat_layer.md` — short brief on the CAM-SIMA ↔ capgen + compatibility layer (for the original ccpp-capgen author). +- `end-to-end-tests/` — the working examples (`instances_advection` + is the newest, exercises everything end-to-end). diff --git a/doc/cam4_fwaut_constituent_order.md b/doc/cam4_fwaut_constituent_order.md new file mode 100644 index 00000000..0e4f042e --- /dev/null +++ b/doc/cam4_fwaut_constituent_order.md @@ -0,0 +1,180 @@ +# cam4 (QPC4) bit-for-bit difference: root cause is constituent registration order + +**Status:** root cause found and **proven**. Decision requested from CAM-SIMA. +**Date:** 2026-06-11 **Author:** D. Heinzeller + +## Executive summary + +The CAM-SIMA test +`SMS_D_Ln9.mpasa120_mpasa120.QPC4.derecho_intel.cam-outfrq_analy_ic_cam4` +(full cam4 physics on the MPAS dynamical core) fails its bit-for-bit (b4b) +comparison against the capgen baseline. The difference is **machine-epsilon +roundoff** — state and flux fields agree to 14–17 significant digits; the +comparison is loud only in RK-microphysics *ratio* diagnostics (e.g. `FWAUT`, +RMS ≈ 4.24e-2), which are ratios of two near-zero autoconversion rates and so +amplify any roundoff. The behavior is identical under GNU and Intel. + +The physics source, `suite_cam4.xml`, and `src/data/registry.xml` are +**byte-identical** between the two builds. The difference is purely in the +generated CCPP caps. We have traced it to a single cause and **proven** it: + +> **capgen registers the advected constituents in a different order than the +> original capgen.** Specifically, `cloud_liquid` and `cloud_ice` +> are swapped. This changes the floating-point summation order in the energy/water +> thermodynamic diagnostics, which the energy fixer then spreads across all columns +> as a tiny, pervasive heating — the source of the b4b difference. + +A one-off patch that forces capgen's advected water species into the +original-capgen order makes **QPC4 bit-for-bit identical** to the baseline. + +## The difference (runtime constituent list, `debug_output = 2`) + +| index | original capgen (baseline) | capgen | +|------:|----------------------------|-----------| +| 1 | **cloud_liquid** (advected) | **cloud_ice** (advected) | +| 2 | **cloud_ice** (advected) | **cloud_liquid** (advected) | +| 3 | water_vapor (advected) | water_vapor (advected) | +| 4–10 | CFC12, O3, CH4, O2, N2O, CFC11, CO2 | CFC12, O2, CH4, CO2, O3, N2O, CFC11 | + +Indices 1–3 are the advected water species; 4–10 are non-advected trace gases. +The advected block is what matters (see mechanism). `water_vapor` is index 3 in +both — the only advected difference is the **cloud_liquid ↔ cloud_ice swap**. + +## Mechanism + +1. `air_composition` builds `thermodynamic_active_species_idx` by walking the + advected constituents in **constituent-index order**. +2. `get_hydrostatic_energy` (`cam_thermo`) sums the water species in that order. + Baseline sums `cloud_liquid + cloud_ice + water_vapor`; capgen sums + `cloud_ice + cloud_liquid + water_vapor`. Same values, **different FP order**. +3. The resulting machine-eps difference in total energy/water is picked up by the + global energy fixer (`check_energy_fix`), which redistributes it as a uniform + heating across all columns. From that point the two runs differ at roundoff + level everywhere, surfacing loudly only in ratio diagnostics like `FWAUT`. + +`air_composition.F90` and `cam_constituents.F90` are byte-identical between the +two builds, so the entire difference originates in the registration order the +generated cap produces. In the CCPP framework, registration order is the +hash-table iteration order in `ccpp_model_constituents_t%lock_table` (advected +packed first) — i.e. an arbitrary, generator-dependent order, not a deliberate +physical ordering. + +## Proof + +Forcing capgen's advected water species into the baseline order +`[cloud_liquid = 1, cloud_ice = 2, water_vapor = 3]` (a flag-guarded one-off +patch in the framework's `ccp_model_const_table_lock`) makes QPC4 reproduce the +ccpp-prebuild baseline **bit-for-bit** (cprnc: all fields identical). This +isolates constituent ordering as the *sole* cause. See section "Artifacts" +below for the full patch. + +## Assessment — neither order is "wrong" + +Both builds register the same constituents with identical properties; the +ordering is not physically meaningful, and the resulting solutions are +roundoff-equivalent and both physically correct. The b4b failure reflects only +that capgen's (arbitrary) order differs from the (equally arbitrary) order +the capgen baseline happened to produce. + +## Decision requested + +To resolve QPC4 (and any other case sensitive to constituent order), we propose: + +1. Give capgen a **deterministic, documented** constituent-registration order + (e.g. water vapor first, with a clear rule for how constituents land in the + array) — replacing today's hash-bucket order. +2. Adopt the new documented order and **re-baseline** the affected CAM-SIMA cases once. + +The temporary proof patch will be removed once the path is agreed. + +## Artifacts + +- **Patch:** Stored as `ccpp_constituent_prop_mod.F90.patch` in the top-level +directory of the `feature/capgen` ccpp-framework branch): +``` +--- capgen/src/ccpp_constituent_prop_mod.F90 ++++ capgen/src/ccpp_constituent_prop_mod.F90 +@@ -1392,6 +1392,17 @@ + type(ccpp_constituent_properties_t), pointer :: cprop + character(len=dimname_len) :: dimname + character(len=*), parameter :: subname = 'ccp_model_const_table_lock' ++ ! === ONE-OFF cam4 constituent-reorder experiment === ++ ! When .true., force the cam4 advected water species into original-capgen ++ ! order [cloud_liquid=1, cloud_ice=2, water_vapor=3] instead of hash-table ++ ! order, to prove the FWAUT b4b diff is driven purely by constituent order. ++ ! Only the 3 cam4 water-species std-names are remapped; everything else keeps ++ ! its normal hash-order index, so other suites are unaffected unless they ++ ! advect exactly these names. Flip to .false. (or delete) to restore. ++ logical, parameter :: l_const_reorder = .true. ++ integer :: const_pos ++ character(len=512) :: sname_reorder ++ ! === end experiment === + + astat = 0 + errcode_local = 0 +@@ -1460,9 +1471,24 @@ + errcode_local = errcode_local + 1 + exit + end if +- call cprop%set_const_index(index_advect, & ++ ! === ONE-OFF cam4 constituent-reorder experiment === ++ const_pos = index_advect ++ if (l_const_reorder) then ++ call cprop%standard_name(sname_reorder, & ++ errcode=errcode, errmsg=errmsg) ++ select case (trim(sname_reorder)) ++ case ('cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water') ++ const_pos = 1 ++ case ('cloud_ice_mixing_ratio_wrt_moist_air_and_condensed_water') ++ const_pos = 2 ++ case ('water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water') ++ const_pos = 3 ++ end select ++ end if ++ call cprop%set_const_index(const_pos, & + errcode=errcode, errmsg=errmsg) +- call this%const_metadata(index_advect)%set(cprop) ++ call this%const_metadata(const_pos)%set(cprop) ++ ! === end experiment === + else + index_const = index_const + 1 + if (index_const > num_vars) then +``` + +- **Run directories (Derecho) Intel:** Because the SIMA baselines change continuously, + - Baseline (original capgen, https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-reference): + - `/glade/derecho/scratch/heinzell/aux_sima_intel_20260614203021/` + - capgen differences to be evaluated against this baseline, because the official baseline changes frequently + - Both the capgen baseline and the capgen test fail for this test: +``` + SMS_Ln9.ne3pg3_ne3pg3_mg37.FKESSLER.derecho_intel.cam-outfrq_se_cslam_multitape (Overall: NLFAIL) details: + FAIL SMS_Ln9.ne3pg3_ne3pg3_mg37.FKESSLER.derecho_intel.cam-outfrq_se_cslam_multitape NLCOMP +``` + - capgen (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen), unpatched (shows the FWAUT diff): + - `/glade/derecho/scratch/heinzell/aux_sima_intel_20260614202951/` with the following `mpasa120_mpasa120.QPC4` test dirs: + - `SMS_Ln9.mpasa120_mpasa120.QPC4.derecho_intel.cam-outfrq_analy_ic_cam4.GC.aux_sima_intel_20260614202951.ORIGINAL_NO_PATCH` + - `SMS_D_Ln9.mpasa120_mpasa120.QPC4.derecho_intel.cam-outfrq_analy_ic_cam4.GC.aux_sima_intel_20260614202951.ORIGINAL_NO_PATCH` + - capgen (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen) + reorder patch (**b4b**): + - `/glade/derecho/scratch/heinzell/aux_sima_intel_20260614202951/` with the following `mpasa120_mpasa120.QPC4` test dirs: + - `SMS_Ln9.mpasa120_mpasa120.QPC4.derecho_intel.cam-outfrq_analy_ic_cam4.GC.aux_sima_intel_20260614202951` + - `SMS_D_Ln9.mpasa120_mpasa120.QPC4.derecho_intel.cam-outfrq_analy_ic_cam4.GC.aux_sima_intel_20260614202951` + +- **Run directories (Derecho) GNU:** Because the SIMA baselines change continuously, + - Baseline (original capgen, https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-reference): + - `/glade/derecho/scratch/heinzell/aux_sima_gnu_20260611123848/` + - capgen differences to be evaluated against this baseline, because the official baseline changes frequently + - Both the capgen baseline and the capgen test fail for this test: +``` + SMS_Ln2.ne3pg3_ne3pg3_mg37.FPHYStest.derecho_gnu.cam-outfrq_hb_vdiff_derecho (Overall: FAIL) details: + FAIL SMS_Ln2.ne3pg3_ne3pg3_mg37.FPHYStest.derecho_gnu.cam-outfrq_hb_vdiff_derecho RUN time=13 + SMS_Ln9.ne3pg3_ne3pg3_mg37.FADIAB.derecho_gnu.cam-outfrq_se_cslam (Overall: FAIL) details: + FAIL SMS_Ln9.ne3pg3_ne3pg3_mg37.FADIAB.derecho_gnu.cam-outfrq_se_cslam RUN time=13 +``` + - capgen (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen), unpatched (shows the FWAUT diff): + - `/glade/derecho/scratch/heinzell/aux_sima_gnu_20260611123837/` with the following `mpasa120_mpasa120.QPC4` test dirs: + - `SMS_Ln9.mpasa120_mpasa120.QPC4.derecho_gnu.cam-outfrq_analy_ic_cam4.GC.aux_sima_gnu_20260611123837.ORIGINAL_NO_PATCH` + - `SMS_D_Ln9.mpasa120_mpasa120.QPC4.derecho_gnu.cam-outfrq_analy_ic_cam4.GC.aux_sima_gnu_20260611123837.ORIGINAL_NO_PATCH` + - capgen (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen) + reorder patch (**b4b**): + - `/glade/derecho/scratch/heinzell/aux_sima_gnu_20260611123837/` with the following `mpasa120_mpasa120.QPC4` test dirs: + - `SMS_Ln9.mpasa120_mpasa120.QPC4.derecho_gnu.cam-outfrq_analy_ic_cam4.GC.aux_sima_gnu_20260611123837` + - `SMS_D_Ln9.mpasa120_mpasa120.QPC4.derecho_gnu.cam-outfrq_analy_ic_cam4.GC.aux_sima_gnu_20260611123837` diff --git a/doc/capgen-ng_review_plan.xlsx b/doc/capgen-ng_review_plan.xlsx new file mode 100644 index 00000000..6e19ab99 Binary files /dev/null and b/doc/capgen-ng_review_plan.xlsx differ diff --git a/doc/capgen_compat_layer.md b/doc/capgen_compat_layer.md new file mode 100644 index 00000000..00dcbd48 --- /dev/null +++ b/doc/capgen_compat_layer.md @@ -0,0 +1,77 @@ +# The CAM-SIMA ↔ capgen compatibility layer — short brief + +*For the original ccpp-capgen author, to orient a feedback pass. +Full reference: `cime_config/capgen_compat/README.md` in the CAM-SIMA +tree. Last revised 2026-06-05.* + +## Why it exists + +capgen is replacing original ccpp-capgen as CAM-SIMA's CCPP code +generator. Rather than rewrite CAM-SIMA's autogen pipeline up front, a +thin **compatibility layer** lets that pipeline keep calling original +capgen's Python surface while capgen does the generation underneath. +`cam_autogen.py`, `generate_registry_data.py`, `write_init_files.py`, +and `hist_config.py` are **unmodified** — they import the facade instead +of original capgen. CAM-SIMA owns this directory; capgen owns nothing +in it. It is **transient scaffolding** (see "Convergence goal"). + +## What the facade reconstructs + +| Original-capgen surface | Rebuilt over (capgen) | File | +|---|---|---| +| `cap_database.host_model_dict()` / `.call_list(phase)` | flat `host_dict` + per-phase `ResolvedArg` lists | `_cap_database.py` | +| per-variable `Var` accessors (`get_prop_value`, `source.ptype`, `array_ref`, `intrinsic_elements`, `call_string`, …) | `HostVarEntry` (host path) / `ResolvedArg` (call-list path) | `_var_wrapper.py` | +| `MetaVar` / `MetadataSection` accessors used by the registry generator | capgen `MetaVar` / `MetadataSection` via monkey-patch | `metadata_table.py` | +| `ParseObject`, `FortranWriter`, the richer `ParseContext` | **vendored verbatim from original capgen** (CAM-SIMA's own scripts use these; capgen does not) | `parse_object.py`, `fortran_write.py`, `parse_source.py` | + +## Three design contracts worth your eyes + +capgen classifies every scheme argument into exactly **one** source — +`control | host | suite | constituent` — on a flat `ResolvedArg` (there +is no `ConstituentVarDict`/scope-chain). The adapter keys on that: + +1. **`source='suite'` is dropped from `call_list`.** These are + interstitials produced and consumed within one suite (they live in + `_data`), so they are not host variables; surfacing them would + trip `write_init_files`' "missing required host variable" check. +2. **`source='constituent'` is mapped to `advected/constituent=True`** so + `write_init_files` routes it through the constituents object (skip + USE-import, skip the IC read). Key detail: the adapter keys on the + **source**, not on a per-arg `is_constituent` flag. capgen now lets + an *unflagged* scheme consume a constituent — base *or* `tendency_of_*` + — because whether a name is a constituent is the **host's** decision; + such consumers carry `is_constituent=False` while still being + framework-supplied. +3. **`type = module` → `type = host`.** capgen renamed your + `type = module`; the shim rewrites it at parse time and records which + tables were "module" so `write_init_files` still gets `ptype='module'` + (allocate + initialise) vs `ptype='host'` (passed via the arg list). + +## Convergence goal (the important framing) + +End state: this directory **does not exist**. CAM-SIMA talks to capgen +through **three CLI utilities** — `ccpp_validator.py`, +`ccpp_capgen.py`, `ccpp_datafile.py` — plus the on-disk +`datatable.xml` contract. A well-defined, feature-equivalent Python +API to these three utilities is also discussed in +`cime_config/capgen_compat/README.md`. No production CAM-SIMA path should +depend on capgen's Python internals; today's `return_state=True` hook handing +back `(host_dict, suite_resolutions)` is scaffolding for this layer only. +The README has a phased retirement plan (A–G) with a measurable LOC drop +per phase. + +## Status + +`kessler`, `rrtmgp`, and `se_cslam` (the full `cam7` suite) build **and +run to completion** on Derecho under **both gnu and intel**, with +bit-comparable results. + +## Feedback we'd value + +1. Does the four-source model (`control/host/suite/constituent`) capture + everything `ConstituentVarDict` did for CAM-SIMA? +2. Are there `cap_database` / `Var` accessors that `write_init_files` or + `generate_registry_data` rely on that we've under- or mis-modeled? +3. Is **CLI + `datatable.xml`** a sufficient convergence interface for + everything CAM-SIMA currently reads out of original-capgen Python + objects — or is there state that has no on-disk equivalent yet? diff --git a/doc/code_walkthrough_DRAFT.md b/doc/code_walkthrough_DRAFT.md new file mode 100644 index 00000000..49760b37 --- /dev/null +++ b/doc/code_walkthrough_DRAFT.md @@ -0,0 +1,532 @@ +# capgen — code walkthrough for prebuild/capgen developers (DRAFT) + +> **Status: temporary draft for the developer walkthrough.** All `file → routine → line` +> anchors were verified against the current tree; line numbers drift, so treat them as +> “go here,” not gospel. Three running examples: a **simple** one +> (`end-to-end-tests/instances/`) used to teach the whole pipeline, an **advanced** +> one (`end-to-end-tests/capgen/`) for the resolver’s harder features, and a +> **constituents** one (`end-to-end-tests/advection/`) for the constituent subsystem. +> Once reviewed, this folds into `doc/DevelopersGuide/`. + +--- + +## 0. Orientation for prebuild/capgen developers + +If you come from **ccpp-prebuild**: there is no Python-templated giant cap and no +`ccpp_prebuild_config.py`. capgen parses metadata and the SDF, **resolves every scheme +argument into an explicit Python object** that records *exactly* where the host data lives +and what (if any) unit/kind/flip transform it needs, then emits Fortran from those objects. + +If you come from **original capgen**: the shape is familiar (metadata → host dict → suite +resolution → caps), but the data model is flatter and the resolution result is a plain +dataclass tree (`SuiteResolution → ResolvedGroup → ResolvedCall → ResolvedArg`) you can +print and inspect. + +The single sentence to keep in mind: + +> **A `ResolvedArg` is the unit of truth.** It stores the host-side access expression +> (`call_expr`) *and* the transform plan (`transform_case` + the forward/backward +> expressions). The emitter does almost no thinking — it just renders `ResolvedArg`s. + +--- + +## 1. The pipeline at a glance + +Everything is orchestrated by `capgen()` in **`ccpp_capgen.py:863`**. + +```mermaid +flowchart TD + A["parse .meta files
parse_metadata_file()
metadata_table.py:1166"] --> B["build flat host dict
build_flat_host_dict()
variable_resolver.py:614"] + A --> C["build scheme store
SchemeStore.build_from()
variable_resolver.py:809"] + D["parse SDF XML
parse_suite_xml_files()
suite_xml.py"] --> E + B --> E["resolve_suite()
suite_resolver.py:2313"] + C --> E + E --> F["SuiteResolution
(groups → calls → args)"] + F --> G["write_group_cap()
group_cap.py:1272
(emit the call string)"] + F --> H["write_suite_data / _types / _cap
write_host_cap, write_datatable"] +``` + +| # | Stage | Routine (file:line) | Produces | +|---|-------|---------------------|----------| +| 1 | Parse `.meta` | `parse_metadata_file` (`metadata_table.py:1166`) → `MetadataTable` (`:940`) / `MetaVar` (`:414`) | per-file tables | +| 2 | Flat host dict | `build_flat_host_dict` (`variable_resolver.py:614`) | `{std_name: HostVarEntry}` | +| 3 | Scheme store | `SchemeStore.build_from` (`variable_resolver.py:809`) | per-scheme ordered arg lists | +| 4 | Parse SDF | `parse_suite_xml_files` (`suite_xml.py`) | suite/group/subcycle/scheme objects | +| 5 | **Resolve** | `resolve_suite` (`suite_resolver.py:2313`) | `SuiteResolution` | +| 6 | **Emit calls** | `write_group_cap` (`group_cap.py:1272`) | `ccpp___cap.F90` | +| 7 | Emit rest | `write_suite_data/_types/_cap`, `write_host_cap`, `write_datatable` | suite data module, host cap, datatable | + +--- + +## 2. The dictionaries — what exists *before* matching + +### 2a. Per-file tables (`MetadataTable` / `MetaVar`) + +`parse_metadata_file` returns one `MetadataTable` per `[ccpp-table-properties]` block; each +holds the `[ccpp-arg-table]` variables as `MetaVar`s (standard_name, local_name, type, kind, +units, dimensions, intent, optional, active, …). This is a faithful in-memory copy of the +`.meta` text — no matching yet. + +### 2b. The flat host dictionary — `host_dict` + +Built once by `build_flat_host_dict` (`variable_resolver.py:614`). It is a flat map keyed by +**standard name**; each value is a `HostVarEntry` (`variable_resolver.py:244`): + +``` +host_dict : { standard_name -> HostVarEntry } + +HostVarEntry +├─ standard_name "data_array2" +├─ local_name "data_array2" +├─ access_path "instance_data(instance_number)%data_array2" ← fully-qualified! +├─ module_name "data" (None for control vars) +├─ type / kind "real" / "kind_phys" +├─ units "m2 s-2" +├─ dimensions ["horizontal_dimension"] +├─ protected / optional / allocatable / active +└─ top_at_one (vertical orientation, for flip detection) +``` + +Key point for prebuild devs: **DDT flattening happens here, at dict-build time.** A host +DDT instance (e.g. `instance_data(number_of_instances)` of type `instance_type`) is walked +recursively (`build_ddt`, `variable_resolver.py:~454`) so that each leaf becomes its own +`HostVarEntry` whose `access_path` already contains the component path and the instance +subscript — e.g. `instance_data(instance_number)%data_array2`. By the time resolution runs, +there are no DDTs left to chase; just standard-name → access-path. + +### 2c. The scheme store + +`SchemeStore.build_from` (`variable_resolver.py:809`) holds, per scheme + phase, the ordered +list of dummy arguments (each a `MetaVar`). This is the *demand* side: “scheme X, run phase, +wants these standard names with these intents/units/dims.” + +### 2d. Snapshot — `instances/` before resolution + +Host side (from `instances/data.meta`), after DDT flattening: + +``` +host_dict = { + "horizontal_dimension" -> ncols (control/host scalar, int) + "number_of_species" -> nspecies + "data_array_all_species" -> instance_data(instance_number)%data_array dims (horiz, species) + "data_array" -> instance_data(instance_number)%data_array(:,2) ← scalar-index sub-var + "data_array2" -> instance_data(instance_number)%data_array2 units m2 s-2 + "data_array_opt" -> instance_data(instance_number)%data_array(:,1) optional, active=(flag_for_opt_array) + "flag_for_opt_array" -> instance_data(instance_number)%opt_array_flag + "instance_number" -> (control var) + "number_of_instances" -> (control var, the instance count) +} +``` + +Demand side (`unit_conv_scheme_1.meta`, run phase): `ccpp_error_message`, +`ccpp_error_code`, `instance_number`, `data_array` (inout, **m**), `data_array2` +(inout, **J kg-1**), `data_array_opt` (inout, **m**, optional). + +Notice the two mismatches the resolver must handle: `data_array2` is **`m2 s-2`** on the host +but **`J kg-1`** in the scheme (a unit transform), and `data_array_opt` is **optional**. + +--- + +## 3. Resolution — matching schemes against the dictionaries + +`resolve_suite` (`suite_resolver.py:2313`) walks the suite in execution order +(groups → subcycles → schemes → phases). For each scheme argument it looks the standard +name up and lands in one of three cases: + +```mermaid +flowchart TD + S["scheme arg: standard_name, intent"] --> Q{"in host_dict?"} + Q -- yes --> H["bind to host/control
source='host'/'control'"] + Q -- no --> Q2{"already a suite_var?"} + Q2 -- yes --> SU["bind to suite-owned var
source='suite'"] + Q2 -- no --> Q3{"intent == out?"} + Q3 -- yes --> P["PROMOTE: create SuiteVar,
add to suite_vars dict
(interstitial)"] + Q3 -- no --> ERR["ERROR: in/inout var
nobody produces
(suite_resolver.py:~1493)"] +``` + +- **Found in host** → `host_dict.get(std)` (`_resolve_single_bound`, `suite_resolver.py:429`). +- **Not found, first use is `intent(out)`** → it’s an interstitial; **promote** it to a + suite-owned variable: a `SuiteVar` (`:964`) is created and added to the running `suite_vars` + dict, so later schemes that read it bind via `source='suite'`. This is capgen’s answer + to prebuild’s “where do interstitials live” — they’re emitted into `ccpp__data.F90`. +- **Not found, first use is `in`/`inout`** → hard error (nobody ever writes it). See the + “undefined intent(out)” discipline — capgen refuses to silently read an unproduced var. + +The two dictionaries in play during resolution: + +| Dict | Lifetime | Keyed by | Value | +|------|----------|----------|-------| +| `host_dict` | built once, read-only | standard_name | `HostVarEntry` | +| `suite_vars` | **grows during resolution** | standard_name | `SuiteVar` | + +So “how are variables resolved between schemes?” → scheme A’s `intent(out)` arg that isn’t a +host variable creates a `SuiteVar`; scheme B later in the same suite, reading the same +standard name, matches that `SuiteVar`. The connection is by **standard name**, and the +storage is `ccpp_suite_data%` (`SuiteVar.access_path`). + +--- + +## 4. Where the resolution is stored — `ResolvedArg` + +This is the crux of your question. Each matched argument becomes a **`ResolvedArg`** +(`suite_resolver.py:1010`). It carries both halves: *where the host data is* and *how to +shuttle it into/out of the scheme*. + +``` +ResolvedArg +├─ scheme_local_name "data_array2" ← keyword in the Fortran call +├─ intent / is_optional / active(_local) +├─ source "host" | "control" | "suite" | "constituent" +├─ host_entry -> HostVarEntry (None if suite-owned) +├─ suite_var -> SuiteVar (None if host) +│ +│ ── WHERE THE DATA IS ─────────────────────────────────────── +├─ base_expr "instance_data(instance_number)%data_array2" +├─ subscript "(:)" or "(lb:ub, 1:nlev)" … +├─ call_expr base_expr + subscript ← the access string +├─ used_dim_std_names {standard names used in the subscript} → drives USE + dummy args +│ +│ ── HOW TO SHUTTLE IT ────────────────────────────────────── +├─ transform_case 1=direct · 2=pointer · 3=transform · 4=pointer+transform +├─ needs_unit/kind/vert flags +├─ unit_forward host→scheme expr (pre-call, intent in/inout) +├─ unit_backward scheme→host expr (post-call, intent out/inout) +├─ temp_name "_l" transform temporary +└─ ptr_name "_p" optional pointer wrapper +``` + +The objects nest exactly like the suite: + +```mermaid +flowchart TD + SR["SuiteResolution (:1333)
suite_vars, init/final calls"] --> RG["ResolvedGroup (:1301)
one per group, phases"] + RG --> RC["ResolvedCall (:1153)
one per scheme invocation"] + RC --> RA["ResolvedArg (:1010)
one per argument"] + RG -.-> RSub["ResolvedSubcycle (:1240)
wraps calls in a do-loop"] +``` + +`ResolvedCall.used_modules` (`:1168`) aggregates `{module: {symbols}}` across its args so the +emitter can write the `use … only:` lines. `write_suite_meta` (`suite_data.py:481`) dumps this +whole tree to a `.meta` for inspection — **the fastest way to see a resolution is to read that +file after a run.** + +### `instances/` resolution snapshot + +| scheme arg | std name | source | `call_expr` | transform_case | +|---|---|---|---|---| +| `instance` | `instance_number` | control | `instance_number` (dummy arg) | 1 (direct) | +| `data_array` | `data_array` | host | `instance_data(instance_number)%data_array(:,2)` | 1 (direct; m=m) | +| `data_array2` | `data_array2` | host | `instance_data(instance_number)%data_array2` | **3** (m2 s-2 ↔ J kg-1) | +| `data_array_opt` | `data_array_opt` | host | `instance_data(instance_number)%data_array(:,1)` | **2** (optional) | + +(`instance_number` is a control var → `module_name = None` → passed as a dummy argument +threaded down from `ccpp_physics_run`, not `use`d.) + +--- + +## 5. Emission — `ResolvedArg` → the call string + +`write_group_cap` (`group_cap.py:1272`) renders each `ResolvedCall`: + +1. **USE statements** from `used_modules` (host/suite modules + symbols). +2. **Pre-call lines** — `_pre_call_lines` (`group_cap.py:575`). +3. **The call** — `=` for every arg. +4. **Post-call lines** — `_post_call_lines` (`group_cap.py:613`). + +The `transform_case` decides everything (the “actual arg” passed is in column 3): + +| case | meaning | pre-call | actual arg | post-call | +|------|---------|----------|------------|-----------| +| 1 | direct | — | `call_expr` | — | +| 2 | optional ptr | `ptr%ptr => call_expr` (or `nullify` if inactive) | `ptr%ptr` | `nullify(ptr%ptr)` | +| 3 | transform | `temp = unit_forward` (in/inout) | `temp` | `call_expr = unit_backward` (out/inout) | +| 4 | ptr + transform | `temp = unit_forward; ptr%ptr => temp` | `ptr%ptr` | `nullify; call_expr = unit_backward` | + +For `instances/`, the emitted group cap (schematically) is: + +```fortran +use data, only: instance_data +... +! data_array2 (case 3): host m2 s-2 -> scheme J kg-1 +data_array2_l = +! data_array_opt (case 2): optional pointer +if (flag_for_opt_array) then + data_array_opt_p%ptr => instance_data(instance_number)%data_array(:,1) +else + nullify(data_array_opt_p%ptr) +end if + +call unit_conv_scheme_1_run( & + instance = instance_number, & ! case 1 + data_array = instance_data(instance_number)%data_array(:,2),& ! case 1 + data_array2 = data_array2_l, & ! case 3 (temp) + data_array_opt = data_array_opt_p%ptr, & ! case 2 (ptr) + errmsg=errmsg, errflg=errflg) + +! data_array2 post: copy back scheme -> host +instance_data(instance_number)%data_array2 = +nullify(data_array_opt_p%ptr) +``` + +That is the whole chain: **`.meta` → `host_dict`/scheme args → `ResolvedArg.call_expr` + +`transform_case` → these emitted lines.** + +> Note the **scheme appears twice** in `instances/` (`unit_conv_scheme_1`, `_2`, `_1`). +> Each appearance is its own `ResolvedCall`; capgen dedups *init/finalize* phases by +> scheme name within a group, but **run** phases emit every appearance. + +--- + +## 6. Running example 1 (simple) — `instances/` end-to-end + +Walk it through the 7 stages: + +1. **Parse** `data.meta` (host + `instance_type` DDT) and the two scheme `.meta`s. +2. **Host dict** — DDT flattened; note `data_array`/`data_array_opt` are scalar-index + sub-views (`data_array(:,2)`, `data_array(:,1)`) of one stored array, and the instance + subscript `(instance_number)` is baked into every `access_path` (§2d). +3. **Scheme store** — `unit_conv_scheme_1/2` run-phase arg lists. +4. **SDF** — one group, `scheme_1`, `scheme_2`, `scheme_1`. +5. **Resolve** — all args hit the host (no promotion here); one unit transform, one optional + (§4 table). +6. **Emit** — §5 listing; plus the host driver loops `do ins = 1, ninstances` passing + `instance = ins`, so the same cap re-runs per instance with the `(instance_number)` + subscript selecting that instance’s slice. +7. **Rest** — `ccpp_instances_data.F90` is essentially empty (nothing promoted); the suite/ + host caps wire `instance_number`/`number_of_instances` as the paired instance control. + +**What this example teaches:** the full parse→dict→resolve→emit spine, DDT flattening, +scalar-index dimensions, control vars, a unit transform (case 3), and an optional arg +(case 2) — with *zero* promotion, so the host↔scheme matching is unobscured. + +--- + +## 7. Running example 2 (advanced) — `capgen/` : what the resolver adds + +Same pipeline; this case exercises the features `instances/` doesn’t. Read it for: + +- **Suite-level promotion (interstitials).** A scheme’s `intent(out)` var that no host table + declares becomes a `SuiteVar` (§3) and is emitted into `ccpp_temp_suite_data.F90`. Trace a + variable that is *not* in `test_host.meta` but is produced by one scheme and consumed by a + later one — that is the `suite_vars` path, and the “variables that should be promoted to + suite level” bullet in the case’s `README.md`. +- **Deeper DDT usage**, including an *undocumented* DDT member — exercises `build_ddt` + recursion and the “don’t require every component to be documented” rule. +- **Non-standard / integer dimensions** and `ccpp_constant_one:N` vs bare `N` — exercises the + subscript builder (`_build_call_subscript`, `suite_resolver.py:479`) and dimension + canonicalisation (`_canonical_dim`, `:761`). +- **Register-phase dimensions** set by a scheme and then used to size module-level + interstitials — the resolver must order phases so the dimension is known before allocation + (`validate_init_dimensions`, called from `capgen()`). +- **Multiple suites & groups + threading** — one `ResolvedGroup` per group, dispatched by the + suite cap’s `select case` on `group_name`. + +Suggested walkthrough move: open `write_suite_meta`’s output for `temp_suite`, find a +promoted variable, and show its `SuiteVar` (no `host_entry`, `access_path = +ccpp_suite_data(1)%…`) next to the two `ResolvedArg`s that produce and consume it. + +--- + +## 8. Running example 3 (constituents) — `advection/` + +Constituents are the one major subsystem the first two examples don’t touch — and the entire +constituent half of `ResolvedArg` (`source='constituent'`, `is_constituent_arg`, +`constituent_module_name`, `constituent_extra_symbols`, `constituent_index_std_names`, +`used_const_dim_std_names`) only comes alive here. `advection/` (cloud liquid/ice tracers) +exercises **three distinct constituent paths**. Suite: `const_indices`, `cld_liq`, +`apply_constituent_tendencies`, `cld_ice`, `apply_constituent_tendencies`. + +### The model (for prebuild developers) + +A **constituent** is a tracer the *host’s dynamical core* owns — water vapor, cloud liquid, +ozone, a chemistry species — that physics reads and updates, together with an optional +**tendency** (the rate of change physics writes back for the dycore to advect it forward). +The crucial difference from an ordinary host variable: the host does **not** hand you a Fortran +array per constituent. The framework owns **one `ccpp_model_constituents_t` object per model +instance**, holding all constituent values (`%vars_layer`), tendencies (`%vars_layer_tend`), +metadata (`%const_metadata`), and the count (`%num_layer_vars`). The resolver translates +standard-name references into subscripts into that object. *(Authoritative deep-dive: +`doc/constituents.md` — “the four rules.”)* + +Three questions prebuild developers always ask: + +**1. What makes a variable a constituent?** Two independent triggers, plus host override: +- a scheme arg carries a **hint attribute** — any of `advected = true`, `constituent = true`, + or a non-default `molar_mass` (`MetaVar.is_constituent`, `metadata_table.py:724`); **or** +- the arg’s **type** is `ccpp_constituent_properties_t` — the register-phase descriptor array + (a separate flag, `is_constituent_arg`); **and** +- constituent-ness is ultimately the **host’s** decision. A scheme that only *reads* a name + need not re-flag it — capgen infers it from the set of names *some* scheme flags (“rule + b”). If the host declares the name as an ordinary variable, that wins + (`design_constituent_host_wins`). + +**2. Where/how are constituents registered?** Exactly one way to declare a *new* one (Rule 1): +a **register-phase** scheme returns an `intent=out, allocatable` array of +`ccpp_constituent_properties_t`, populating each entry via +`%instantiate(std_name=…, units=…, vertical_dim=…, advected=…, …)`. The framework collects +every register scheme’s array and merges them into each instance’s constituent object at +`ccpp_register_constituents`. You **cannot** create a base constituent in a physics phase — +that’s a hard error (Rule 4). + +**3. Are they always tracer + tendency + index triples?** **No** — the pieces are independent: +- the **base constituent** (the tracer): registered once, stored in `%vars_layer`, read via + `index_of_`; +- an **optional tendency**: a *separate* arg whose standard name starts with `tendency_of_`, + stored in `%vars_layer_tend`, implicitly tied to the base of the same name (Rule 3) — a + scheme emits one only if it has a tendency to give; +- the **index** `index_of_`: not something you declare — the framework derives it and fills + it at init via `%const_index()`; the value is identical for every instance. + +So a constituent is *“one registered base + zero-or-more tendency references + a framework +index,”* not a fixed triple. + +**How standard names map to storage** (this is what the resolver emits): + +| Scheme arg references… | Resolves to | +|---|---| +| a base constituent (by name, via its index) | `ccpp_model_constituents_obj(inst)%vars_layer(, index_of_)` | +| `tendency_of_` | `…%vars_layer_tend(, index_of_)` | +| `ccpp_constituents` | `…%vars_layer(:,:,:)` (whole array) | +| `ccpp_constituent_tendencies` | `…%vars_layer_tend(:,:,:)` | +| `number_of_ccpp_constituents` | `…%num_layer_vars` (scalar count) | +| `index_of_` | module-level `integer :: index_of_` | + +Now, how each of these shows up in `advection/`: + +```mermaid +flowchart LR + R["register phase
cld_liq_register"] -->|"dyn_const :
ccpp_constituent_properties_t
(intent out, allocatable)"| FW["framework builds the
per-instance constituent object"] + FW --> IDX["index_of_<X> symbols
+ number_of_ccpp_constituents"] + IDX --> RUN["run phase
base(:,:,index_of_…) · ccpp_constituents(:,:,:)"] +``` + +### 8.1 Registration — the `ccpp_constituent_properties_t` argument + +`cld_liq_register` (a **register**-phase entry) declares: + +``` +[ dyn_const ] + standard_name = dynamic_constituents_for_cld_liq + type = ccpp_constituent_properties_t + dimensions = (:) + intent = out + allocatable = true +``` + +The resolver flags this `is_constituent_arg = True` and — unlike every other not-in-host +`intent(out)` variable — **does not promote it to a `SuiteVar`**. It is a local temporary the +scheme allocates and fills with constituent descriptors (`suite_resolver.py:~1570`). The +framework gathers each register scheme’s `dyn_const` array to build the model’s constituent +set and the per-instance constituent object. + +> Contrast with §3: a normal not-in-host `intent(out)` → `SuiteVar` (suite-owned data). A +> `ccpp_constituent_properties_t` `intent(out)` → *local temp, collected by the framework*. +> This special-case is the one exception to the promotion rule. + +### 8.2 Flagging a produced constituent + +In `cld_liq_run`: + +``` +[ cld_liq_array ] standard_name = cloud_liquid_dry_mixing_ratio advected = .true. intent=inout +[ cld_liq_tend ] standard_name = tendency_of_cloud_liquid_dry_mixing_ratio constituent = True intent=out +``` + +`advected` / `constituent` (non-default) set `ResolvedArg.is_constituent = True`. The backing +store is the framework constituent array, reached by index (next). + +### 8.3 Indexing one constituent — `index_of_` (`source='constituent'`) + +Schemes that touch a single constituent slice declare an access like: + +``` +[ q(:,:,index_of_water_vapor_specific_humidity) ] + standard_name = water_vapor_specific_humidity +``` + +`q` is the constituent backing array; `index_of_water_vapor_specific_humidity` is a +per-constituent index symbol. This is the `source='constituent'` path, and the `ResolvedArg` +records: + +| field | value | purpose | +|---|---|---| +| `constituent_module_name` | suite cap module | module to `use` | +| `constituent_extra_symbols` | `{index_of_water_vapor_specific_humidity}` | symbols to `use` (the index integers) | +| `constituent_index_std_names` | `{water_vapor_specific_humidity}` | the **real** standard name, kept verbatim | + +Why keep the real name separately? The Fortran `index_of_*` symbol is **mangled** to fit the +63-char identifier limit (`_index_symbol_name`, `suite_resolver.py:148`), so its suffix is not +a reliable source of the standard name. At init, the framework fills each index via +`%const_index()`. + +> **Host-wins:** if the host itself declares the `index_of_*` / framework names, the resolver +> short-circuits to ordinary host-arg resolution (the constituent path is skipped). That’s the +> `design_constituent_host_wins` rule. + +### 8.4 The whole constituent axis — `number_of_ccpp_constituents` + +`apply_constituent_tendencies_run` takes the full arrays: + +``` +[ const ] standard_name = ccpp_constituents dims (horizontal, vertical, number_of_ccpp_constituents) +[ const_tend ] standard_name = ccpp_constituent_tendencies dims (horizontal, vertical, number_of_ccpp_constituents) +``` + +`number_of_ccpp_constituents` is a **framework count dimension** — the host never declares it +as a scalar; its value comes from the per-instance constituent object at runtime. The resolver +files it under `ResolvedArg.used_const_dim_std_names` (note: *not* `used_dim_std_names`), so it +produces **no** `use`/dummy-arg, but it *is* surfaced by the host introspection routines. The +emitted call passes the whole axis (`:`) on that dimension. + +### 8.5 The error suite + +`cld_suite_error.xml` swaps in `dlc_liq` — a scheme whose `ccpp_constituent_properties_t` +setup is wrong — so the resolver’s constituent error reporting can be demonstrated live (a bad +constituent declaration is rejected, not silently mis-wired). + +### Callout — combining with instances (`instances_advection/`) + +`instances_advection/` is `advection/` **plus** multiple instances. The *only* delta: the +constituent buffer is dimensioned `(number_of_instances)` — a **wrapper-DDT array, one +constituent object per instance** — so per-instance `set_const_index` calls don’t collide +(the per-instance dynamic-constituent buffer). The constituent resolution itself is identical +to §8.1–8.4. Use it only when the audience needs the multi-instance constituent story. + +--- + +## 9. How to follow along live + +- **Run it:** point `capgen()` / `ccpp_capgen.py` at the example’s `.meta` + SDF and inspect + the generated `ccpp_*_cap.F90`, `ccpp_*_data.F90`, and `datatable.xml`. +- **Read the resolution:** `write_suite_meta` (`suite_data.py:481`) emits the resolved suite as + a `.meta` — the cleanest dump of `SuiteResolution`. +- **`--trace`:** regenerate with `--trace` (or flip `logical, parameter :: trace = .true.` in + one cap) to get `CCPP TRACE ` lines at runtime — useful for seeing the *call order* the + resolution produced. +- **Unit tests as specs:** `unit-tests/test_suite_resolver.py`, `test_variable_resolver.py`, + `test_group_cap.py` are small, readable assertions about exactly these structures. + +--- + +## Appendix — `file → routine → line` quick reference + +| Concept | Routine | File:line | +|---|---|---| +| Orchestrator | `capgen` | `ccpp_capgen.py:863` | +| Load metadata | `_load_metadata_files` | `ccpp_capgen.py:637` | +| Parse `.meta` | `parse_metadata_file` | `metadata/metadata_table.py:1166` | +| Parsed table / var | `MetadataTable` / `MetaVar` | `metadata/metadata_table.py:940` / `:414` | +| Host dict entry | `HostVarEntry` | `metadata/variable_resolver.py:244` | +| Build host dict | `build_flat_host_dict` | `metadata/variable_resolver.py:614` | +| DDT flatten | `build_ddt` | `metadata/variable_resolver.py:~454` | +| Scheme store | `SchemeStore.build_from` | `metadata/variable_resolver.py:809` | +| Parse SDF | `parse_suite_xml_files` | `generator/suite_xml.py` | +| **Resolve suite** | `resolve_suite` | `generator/suite_resolver.py:2313` | +| Resolve one bound | `_resolve_single_bound` | `generator/suite_resolver.py:429` | +| Build subscript | `_build_call_subscript` | `generator/suite_resolver.py:479` | +| Suite-owned var | `SuiteVar` | `generator/suite_resolver.py:964` | +| **Resolved arg** | `ResolvedArg` | `generator/suite_resolver.py:1010` | +| Resolved call/group/suite | `ResolvedCall` / `ResolvedGroup` / `SuiteResolution` | `:1153` / `:1301` / `:1333` | +| **Emit group cap** | `write_group_cap` | `generator/group_cap.py:1272` | +| Pre/post transform | `_pre_call_lines` / `_post_call_lines` | `generator/group_cap.py:575` / `:613` | +| Dump resolution | `write_suite_meta` | `generator/suite_data.py:481` | diff --git a/doc/constituents.md b/doc/constituents.md new file mode 100644 index 00000000..775d653b --- /dev/null +++ b/doc/constituents.md @@ -0,0 +1,1063 @@ +# CCPP capgen — Constituents Reference + +*Last revised: 2026-05-13.* + +This document is the authoritative reference for **constituent variables** in +capgen — what they are, how scheme authors declare them in metadata, what +the host model has to do to plumb them through, what the generator emits, and +how the per-instance lifecycle works. + +> If you are migrating a host or scheme from the original capgen, jump to +> [§9 Differences from original capgen](#9-differences-from-original-capgen) +> first. + +--- + +## Table of Contents + +1. [What is a constituent?](#1-what-is-a-constituent) +2. [The four rules (scheme-author conventions)](#2-the-four-rules-scheme-author-conventions) +3. [Required host metadata + Fortran](#3-required-host-metadata--fortran) +4. [Host-side lifecycle (call sequence)](#4-host-side-lifecycle-call-sequence) +5. [Public API reference](#5-public-api-reference) +6. [Generated code structure](#6-generated-code-structure) +7. [Multi-instance design](#7-multi-instance-design) +8. [Limitations and gotchas](#8-limitations-and-gotchas) +9. [Differences from original capgen](#9-differences-from-original-capgen) +10. [Worked example](#10-worked-example) + +--- + +## 1. What is a constituent? + +A **constituent** is a model variable owned by the host's dynamical core (or +its constituent infrastructure) that is read and updated by physics schemes — +typically a tracer / mass mixing ratio (water vapor, cloud liquid, ozone, +chemistry species) — together with its **tendency**, the rate of change that +physics writes back so the dycore can advect/integrate it forward. + +In capgen, the constituent layer has three concerns: + +1. **Registration** — declaring at model startup which constituents exist + (their standard name, units, vertical layout, advection flag, …). +2. **Storage** — the framework owns one `ccpp_model_constituents_t` DDT per + host instance (see [§7](#7-multi-instance-design)) which holds the + constituent values (`%vars_layer`), tendencies (`%vars_layer_tend`), and + metadata (`%const_metadata`). +3. **Access** — schemes reference constituents by standard name in their + metadata; the resolver translates those references to + `ccpp_model_constituents_obj(inst)%vars_layer(slice, index_of_)` + subscripts at code-gen time. + +All constituent state lives in **one generated module**: +`ccpp_host_constituents.F90` (one per generator run, emitted only when at +least one suite touches constituent state). Public symbols from this module +are also re-exported by `_ccpp_cap` (the per-host static API; filename +and module name derived from `--host-name`), so most host code only needs + +```fortran +use _ccpp_cap, only: ccpp_register_constituents, ccpp_initialize_constituents, & + ccpp_constituents_array, ccpp_const_get_index, ... +``` + +--- + +## 2. The four rules (scheme-author conventions) + +These four rules govern every scheme-arg metadata pattern related to +constituents. They derive from a 2026-05-11 audit of all 12 cam-sima scheme +metadata files that touch constituent attributes. + +### Rule 1 — Register a new constituent (register phase) + +A scheme that creates a new constituent declares it in the **register** +phase via an `intent=out, allocatable` array of +`ccpp_constituent_properties_t`: + +``` +[ccpp-arg-table] + name = my_scheme_register + type = scheme +[ dyn_const ] + standard_name = dynamic_constituents_for_my_scheme + long_name = per-scheme constituent array + units = none + dimensions = (:) + type = ccpp_constituent_properties_t + allocatable = True + intent = out +[ errmsg ] + ... +[ errflg ] + ... +``` + +The scheme's Fortran register routine `allocate`s this array, populates +each entry via `%instantiate(std_name=..., long_name=..., units=..., +vertical_dim=..., advected=..., ...)` and returns it. The framework +captures every register-phase scheme's array, packs them into a per-suite +buffer (`_dynamic_constituents`), and merges them into each +host-instance's constituent object during `ccpp_register_constituents`. + +This is the **only path** for declaring a new constituent. + +### Rule 2 — Consume a base constituent (any physics phase) + +A scheme that reads (or reads + writes) an existing base constituent +declares the variable with `is_constituent` set (any of `advected`, +`constituent`, or `molar_mass` non-default) and `intent=in` or `intent=inout`: + +``` +[ cldliq ] + standard_name = cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = in ! or inout + advected = true +``` + +The resolver translates this scheme arg to +`ccpp_model_constituents_obj()%vars_layer(, index_of_)` +in the generated group cap. No host metadata declaration is needed for +the variable. + +**Consumers need not re-flag (rule b, 2026-06-05).** Whether a standard +name is a constituent or an ordinary variable is the **host's** decision +(CAM-SIMA exposes water vapor as a constituent; CCPP-SCM may expose the +same name as an ordinary host variable), so a scheme that only **reads** +a constituent — the base species, or a `tendency_of_` — does **not** +repeat the `advected` / `constituent` flag. capgen infers +constituent-ness for an unflagged `intent=in/inout` consumer from the +scheme-metadata-wide set of names *some* scheme flags +(`VariableResolver.constituent_stdnames()`): an unflagged read of the +base resolves to `%vars_layer(...)`, an unflagged read of +`tendency_of_` to `%vars_layer_tend(..., index_of_)` — the same +column a tendency producer (Rule 3) wrote. **Host / earlier-suite +provision wins**: if the host declares the name, or an earlier scheme +already produced it as an ordinary variable, normal host/suite +resolution takes over. (This is what lets the CAM-SIMA `cam7` +`sima_diagnostics` schemes read `tendency_of_water_vapor_…` that the +convection schemes produce.) + +### Rule 3 — Produce a tendency (any physics phase) + +A scheme that writes a constituent tendency declares the variable with +`is_constituent` set, `intent=out`, and a standard name that **starts +with `tendency_of_`**: + +``` +[ tend_cldliq ] + standard_name = tendency_of_cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water + units = kg kg-1 s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = out + constituent = true +``` + +The resolver translates this scheme arg to +`ccpp_model_constituents_obj()%vars_layer_tend(, index_of_)` +where `` is the std_name with the `tendency_of_` prefix stripped. +The tendency variable is implicitly tied to the base constituent of the +same name. + +### Rule 4 — Mismatched combinations are hard errors + +One combination is rejected by the resolver at code-gen time: + +| Mismatch | Error | +|---|---| +| `is_constituent=True` + `intent=out` + std_name does NOT start with `tendency_of_` | *"Physics phases may only produce constituent tendencies; new base constituents must be declared via a `ccpp_constituent_properties_t` argument in a register-phase scheme."* | + +> **Changed 2026-06-05:** consuming a constituent **tendency** +> (`intent=in/inout` on a `tendency_of_*` standard name) is **no longer** +> an error. It resolves to `%vars_layer_tend(..., index_of_)` — the +> same column a tendency producer writes — so a diagnostics scheme can +> read a tendency another scheme produced. See "Consumers need not +> re-flag (rule b)" under Rule 2. + +### Direct framework-array access + +A scheme may also access the framework's bulk arrays directly by +declaring an arg with one of these standard names: + +| Standard name | Maps to | +|---|---| +| `ccpp_constituents` | `ccpp_model_constituents_obj()%vars_layer` (3D) | +| `ccpp_constituent_tendencies` | `ccpp_model_constituents_obj()%vars_layer_tend` (3D) | +| `ccpp_constituent_properties` | `ccpp_model_constituents_obj()%const_metadata` (1D of `ccpp_constituent_prop_ptr_t`) | +| `number_of_ccpp_constituents` | `ccpp_model_constituents_obj()%num_layer_vars` (scalar integer) | +| `index_of_` | module-level `integer :: index_of_` (no per-instance access — the index is identical for every instance) | + +The trailing dimension `number_of_ccpp_constituents` in a 3D scheme arg +is emitted as `:` (whole-axis slice). + +--- + +## 3. Required host metadata + Fortran + +### Host metadata (`type=host` table) + +The host **must** declare: + +``` +[ ] + standard_name = number_of_instances + units = count + dimensions = () + type = integer +``` + +… **only when the host actually wants multi-instance support**. When +absent, every per-instance allocation falls back to size `1` and the +host effectively runs single-instance. + +The host **does not** need to declare: + +- `ccpp_model_constituents_object` — the constituent object is owned + by the generator (in `ccpp_host_constituents`); the host doesn't + declare it in metadata. +- `ccpp_constituents`, `ccpp_constituent_tendencies`, + `ccpp_constituent_properties`, `number_of_ccpp_constituents`, + `index_of_` — all auto-provided by the generator. + +#### Host metadata wins over auto-provisioning + +If the host **does** declare any of the framework-named standard +names above as a regular host variable, the resolver uses the host's +declaration instead of auto-provisioning. This matters for legacy +hosts (GFS / SCM) that own their own tracer indices: + +```meta +[ ntcw ] + standard_name = index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array + units = index + type = integer + protected = True + dimensions = () +``` + +A scheme arg requesting the same standard name resolves to the host's +short local name (`ntcw`), not a parallel module-level integer in +`ccpp_host_constituents` named after the full standard name (which +would blow the Fortran 63-character identifier limit). Auto-provisioning +only fires for framework-named standard names the host has **not** +claimed. + +### Host control-table requirements + +The host's `type=control` table must declare: + +``` +[ ] + standard_name = instance_number + units = 1 + dimensions = () + type = integer +``` + +… so the framework signature knows the index for per-instance state. +Same caveat as `number_of_instances` — required only when multi-instance +is wanted. + +### Host Fortran code + +The host's Fortran code only needs to: + +1. Maintain its own `integer :: ` for `number_of_instances` + in a module that's USE'd by the generator. (Same module that owns + the metadata.) +2. Build its **host constituents** array (water vapor, ozone, etc. — + the constituents that the host model owns directly, separately from + any scheme-registered ones). Pass this to + `ccpp_register_constituents`. + +The host does **not** need to allocate or own a +`type(ccpp_model_constituents_t)` variable. + +--- + +## 4. Host-side lifecycle (call sequence) + +``` + ┌─ host startup ─┐ + │ + ▼ + ┌──────────────────────────────────────┐ + │ for each instance: │ + │ ccpp_register(suite_name, │ + │ errmsg, errflg, │ + │ instance_number) │ ─── per-instance ───┐ + └──────────────────────────────────────┘ │ + │ │ + ▼ │ + ┌──────────────────────────────────────┐ │ + │ allocate host_constituents(:) │ │ + │ host_constituents(1)%instantiate( │ ─── once ─────────┘ + │ std_name='water_vapor_specific_humidity', ...) │ + │ ... │ │ + └──────────────────────────────────────┘ │ + │ │ + ▼ │ + ┌──────────────────────────────────────┐ │ + │ for each instance: │ │ + │ ccpp_register_constituents( │ │ + │ host_constituents, │ │ + │ instance_number, errflg, errmsg) │ ─── per-instance ──┤ + └──────────────────────────────────────┘ │ + │ │ + ▼ │ + ┌──────────────────────────────────────┐ │ + │ for each instance: │ │ + │ ccpp_initialize_constituents( │ │ + │ ncols, num_layers, │ │ + │ instance_number, errflg, errmsg) │ ─── per-instance ──┤ + └──────────────────────────────────────┘ │ + │ │ + ▼ │ + ┌──────────────────────────────────────┐ │ + │ for each instance: │ │ + │ ccpp_init(suite_name, │ │ + │ errmsg, errflg, │ │ + │ instance_number) │ ─── per-instance ──┤ + └──────────────────────────────────────┘ │ + │ │ + ▼ (model time-stepping) │ + ┌──────────────────────────────────────┐ │ + │ ccpp_physics_*(...) │ ─── per-instance ──┤ + └──────────────────────────────────────┘ │ + │ │ + ▼ (host shutdown) │ + ┌──────────────────────────────────────┐ │ + │ for each instance: │ │ + │ ccpp_deallocate_dynamic_constituents( │ + │ instance_number) │ ─── per-instance ──┤ + │ ccpp_final(suite_name, │ │ + │ errmsg, errflg, │ │ + │ instance_number) │ │ + └──────────────────────────────────────┘ │ + │ + ┌────────────────────────────────────────┘ + │ ◀── last-to-leave dealloc fires + │ automatically inside the per-instance + │ calls when the final instance finishes. + ▼ +``` + +### Important ordering rules + +- `ccpp_register_constituents` **must** be called *after* `ccpp_register` + (per instance). The latter populates the per-suite dynamic-constituent + buffers via `_register`; the former merges them into the + per-instance constituent object. +- `ccpp_initialize_constituents` **must** be called *after* + `ccpp_register_constituents` (per instance). It calls `%lock_data` + on the per-instance object — which can only happen once + `%lock_table` has fired (which `ccpp_register_constituents` does). +- Physics phases (`ccpp_init`, `ccpp_physics_run`, etc.) require + the constituent state to be locked + bound (i.e., + `ccpp_initialize_constituents` already called). +- `ccpp_deallocate_dynamic_constituents` is per-instance with + last-to-leave teardown. Once the last instance calls it, the shared + per-suite buffers and the constituent object array are deallocated + automatically. + +### Built-in constituents vs scheme-registered constituents + +`ccpp_register_constituents` takes one explicit argument: an array of +`ccpp_constituent_properties_t` describing the **host's own constituents** +(typically water vapor and any other tracers the dycore carries +intrinsically). The framework then merges those entries with every +suite's per-suite dynamic-constituent buffer (populated during +`ccpp_register` from each register-phase scheme's output). + +Pass an empty (zero-size) array if the host has no built-in constituents +of its own. + +--- + +## 5. Public API reference + +All routines below live in `ccpp_host_constituents` and are also +re-exported from `_ccpp_cap` for convenience. The dummy-argument +name `instance_number` is the **standard name**; the actual emitted +dummy uses the host's local name for it (typically also +`instance_number` or `inst_num`). + +### `ccpp_register_constituents(host_constituents, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `host_constituents` | `type(ccpp_constituent_properties_t), target, intent(in) :: (:)` | Host-owned constituent declarations (water vapor, etc.). May be zero-size. | +| `instance_number` | `integer, intent(in)` | Per-instance index. | +| `errflg` | `integer, intent(out)` | Error flag (0 = success). | +| `errmsg` | `character(len=*), intent(out)` | Error message. | + +**Effect**: +- On the first call across instances, allocates + `ccpp_model_constituents_obj(number_of_instances)`. +- Calls `obj(instance_number)%initialize_table(num_consts)` where + `num_consts = size(host_constituents) + sum(size(_dynamic_constituents))`. +- Iterates `host_constituents` first, then every suite's + `_dynamic_constituents` buffer, calling + `obj(instance_number)%new_field(const_prop, errcode=errflg, errmsg=errmsg)` + for each entry. +- Calls `obj(instance_number)%lock_table(...)`. + +**Preconditions**: every `_register` call (across all suites) for +this instance has already happened (so the per-suite buffers are +populated). + +### `ccpp_initialize_constituents(ncols, num_layers, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `ncols` | `integer, intent(in)` | Horizontal extent for this instance's chunk. | +| `num_layers` | `integer, intent(in)` | Vertical layer count. | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +**Effect**: +- Calls `obj(instance_number)%lock_data(ncols, num_layers, ...)` — + allocates `obj(inst)%vars_layer` and `%vars_layer_tend` arrays. +- Registers a singleton pointer with + `ccpp_scheme_utils.ccpp_initialize_constituent_ptr(obj(inst))` so + cam-sima schemes that call `ccpp_constituent_index` see the + constituent table. **First instance wins** — see + [§8 Limitations](#8-limitations-and-gotchas). +- Queries `obj(instance_number)%const_index(index_of_, '', ...)` for + every constituent `` known at code-gen time; populates the + module-level integer `index_of_`. These integers are identical + across instances; the last call to set them wins (benign — the + constituent table is the same per instance). + +**Preconditions**: `ccpp_register_constituents` has been called for this +instance. + +### `ccpp_is_scheme_constituent(var_name, constituent_exists, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `var_name` | `character(len=*), intent(in)` | Standard name to query. | +| `constituent_exists` | `logical, intent(out)` | True iff *var_name* matches one of the constituent std names known to capgen at code-gen time. | +| `errflg` / `errmsg` | `intent(out)` | | + +**No `instance_number`** — the data lookup is against the module-level +`character(len=N), parameter :: ccpp_model_const_stdnames(K)` array +(compile-time constant, identical across instances). + +### `ccpp_number_constituents(num_flds, advected, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `num_flds` | `integer, intent(out)` | Constituent count returned. | +| `advected` | `logical, optional, intent(in)` | If `.true.`, count advected only. | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +Wraps `obj(instance_number)%num_constituents(num_flds, advected=advected, ...)`. + +> Even though every `obj(i)` returns the same count (registration is +> identical across instances), `instance_number` is part of the +> signature so the caller can guarantee they're querying an +> already-locked instance. Useful for hosts that lifecycle one +> instance at a time. + +### `ccpp_gather_constituents(const_array, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `const_array` | `real(kind=kind_phys), intent(out) :: (:,:,:)` | Destination buffer for constituent values. | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +Wraps `obj(instance_number)%copy_in(const_array, ...)`. Use this to +pull the per-instance constituent values into a host-side array. + +### `ccpp_update_constituents(const_array, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `const_array` | `real(kind=kind_phys), intent(in) :: (:,:,:)` | Source buffer with updated constituent values. | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +Wraps `obj(instance_number)%copy_out(const_array, ...)`. Use this to +push host-side updates back into the per-instance constituent object. + +### `ccpp_const_get_index(stdname, const_index, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `stdname` | `character(len=*), intent(in)` | Constituent standard name to look up. | +| `const_index` | `integer, intent(out)` | Returned index into the constituent array (or `int_unassigned` on miss). | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +Wraps `obj(instance_number)%const_index(standard_name=stdname, +index=const_index, ...)`. For constituents whose std names are known +at code-gen time, prefer using the module-level `index_of_` integer +directly (no call needed; it's bound during +`ccpp_initialize_constituents`). + +### `ccpp_constituents_array(instance_number) result(const_ptr)` + +Returns `real(kind=kind_phys), pointer :: const_ptr(:,:,:)` → +`obj(instance_number)%field_data_ptr()`. + +### `ccpp_advected_constituents_array(instance_number) result(const_ptr)` + +Returns `real(kind=kind_phys), pointer :: const_ptr(:,:,:)` → +`obj(instance_number)%advected_constituents_ptr()`. Subset of the +full constituent array containing only those flagged `advected=.true.`. + +### `ccpp_model_const_properties(instance_number) result(const_ptr)` + +Returns `type(ccpp_constituent_prop_ptr_t), pointer :: const_ptr(:)` → +`obj(instance_number)%constituent_props_ptr()`. + +### `ccpp_deallocate_dynamic_constituents(instance_number)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `instance_number` | `integer, intent(in)` | | + +**Per-instance reset + last-to-leave teardown**: +1. `obj(instance_number)%reset()` — unlocks the table for this instance. +2. Iterates every `obj(i)` and checks `%const_props_locked()`. If any + instance is still locked, the routine returns. +3. If **every** instance has been reset (none still locked), the routine + tears down the shared state: + - Deallocates every `_dynamic_constituents` buffer. + - Deallocates `ccpp_model_constituents_obj(:)`. + - Resets every `index_of_` integer to 0. + +The host should call this for every instance that successfully called +`ccpp_register_constituents`. + +--- + +## 6. Generated code structure + +When any suite touches constituent state, capgen emits one extra +module per generator run: **`ccpp_host_constituents.F90`**. + +### Module declarations + +```fortran +module ccpp_host_constituents + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: & + ccpp_model_constituents_t, & + ccpp_constituent_properties_t, & + ccpp_constituent_prop_ptr_t + + implicit none + private + + ! ----- public state ---------------------------------------------------- + public :: ccpp_model_constituents_obj + public :: index_of_ ! one per known constituent std name + public :: index_of_ + public :: ccpp_model_const_stdnames ! parameter array + + ! ----- public routines (also re-exported from _ccpp_cap) -------- + public :: ccpp_register_constituents + public :: ccpp_initialize_constituents + public :: ccpp_is_scheme_constituent + public :: ccpp_number_constituents + public :: ccpp_gather_constituents + public :: ccpp_update_constituents + public :: ccpp_const_get_index + public :: ccpp_constituents_array + public :: ccpp_advected_constituents_array + public :: ccpp_model_const_properties + public :: ccpp_deallocate_dynamic_constituents + public :: _dynamic_constituents ! one per suite with register-phase producers + public :: _dynamic_constituents + + ! ----- module-level state --------------------------------------------- + type(ccpp_model_constituents_t), target, allocatable :: ccpp_model_constituents_obj(:) + type(ccpp_constituent_properties_t), allocatable, target :: _dynamic_constituents(:) + type(ccpp_constituent_properties_t), allocatable, target :: _dynamic_constituents(:) + integer :: index_of_ = 0 + integer :: index_of_ = 0 + character(len=N), parameter :: ccpp_model_const_stdnames(K) = (/ & + ' ', & + ' ' /) + +contains + ! ... routines as documented in §5 ... +end module ccpp_host_constituents +``` + +### Suite-cap responsibilities + +`ccpp__cap.F90` does NOT own constituent state. Its +`_register` routine packs each register-phase scheme's +constituent array into the suite's `_dynamic_constituents` +buffer (USE'd from `ccpp_host_constituents`): + +```fortran +! Outer wrapper sized to number_of_instances on first call (any instance). +if (.not. allocated(_dynamic_constituents)) then + allocate(_dynamic_constituents(number_of_instances)) +end if + +! Single pass: call each scheme's _register EXACTLY ONCE and append its +! returned array to THIS instance's slot. +allocate(_dynamic_constituents(inst)%items(0)) +call _register(dyn_const=scheme_consts, ...) +if (errflg /= 0) return +_dynamic_constituents(inst)%items = & + [_dynamic_constituents(inst)%items, scheme_consts] +deallocate(scheme_consts) +! ... one block like the above per constituent-registering scheme ... +``` + +Each instance owns its own `%items` slot (the per-instance buffer, so +`ccpp_register_constituents` can `set_const_index` independently per +instance); the suite state-machine guard ensures each instance populates +it exactly once. Each scheme's `_register` is called **exactly once** — +it may safely allocate persistent module state (the earlier two-pass +count+copy called it twice). The host-wide merge happens in +`ccpp_register_constituents`. + +### Group-cap call sites + +`ccpp___cap.F90` USE's the constituent symbols it needs +from `ccpp_host_constituents`: + +```fortran +use ccpp_host_constituents, only: ccpp_model_constituents_obj, & + index_of_cloud_liquid_water_mixing_ratio +``` + +… and emits scheme call sites with the per-instance access expression: + +```fortran +call cld_liq_run( & + ... + cldliq=ccpp_model_constituents_obj(inst_num)%vars_layer(lb:ub, 1:nlev, & + index_of_cloud_liquid_water_mixing_ratio), & + tend_cldliq=ccpp_model_constituents_obj(inst_num)%vars_layer_tend(lb:ub, 1:nlev, & + index_of_cloud_liquid_water_mixing_ratio), & + ...) +``` + +The `instance_number` dummy is auto-injected into the group-cap +subroutine signatures by `_extra_dim_ctrl_entries` because the +resolver adds `instance_number` to every constituent arg's +`used_dim_std_names`. + +### Framework F90 dependencies + +`ccpp_host_constituents.F90` and the suite caps depend on these +framework files (listed under `` in `datatable.xml`): + +| File | Why | +|---|---| +| `ccpp_constituent_prop_mod.F90` | Provides `ccpp_model_constituents_t`, `ccpp_constituent_properties_t`, `ccpp_constituent_prop_ptr_t`. | +| `ccpp_hashable.F90` | Transitive dep of `ccpp_constituent_prop_mod`. | +| `ccpp_hash_table.F90` | Transitive dep. | +| `ccpp_scheme_utils.F90` | Provides `ccpp_initialize_constituent_ptr` + `ccpp_constituent_index` (used by cam-sima rrtmgp / mmm schemes). | + +The host's CMake should query `ccpp_datafile.py --utility-files` to +get the absolute paths to these files at the right output location. + +--- + +## 7. Multi-instance design + +In capgen, **per-instance state** means: each "instance" (typically +an OpenMP team / chunk-domain partition) has its own copy of the +state arrays, indexed by `instance_number ∈ [1, number_of_instances]`. + +### What's per-instance + +| State | Storage | +|---|---| +| Constituent values + tendencies | `ccpp_model_constituents_obj(:)` — one DDT per instance | +| Suite-level state machine | `ccpp_suite_state(:)` — declared in each suite cap | +| Suite-owned data | `ccpp_suite_data(:)` — declared in each suite-data module | +| Group-level state machine | `ccpp_group_state(:)` — declared in each group cap | + +### What's shared across instances + +| State | Reason | +|---|---| +| `_dynamic_constituents(:)` per-suite buffers | Registration is identical per instance — populated by the first instance to call `_register` and reused by the rest | +| `index_of_` integers | The constituent table is identical per instance, so the indices are too | +| `ccpp_model_const_stdnames` parameter array | Compile-time constant | + +### Sizing + +`number_of_instances` is the single source of truth. The host declares +it in metadata + Fortran; the generator USE's it from the host module +wherever per-instance allocation happens. See the prior memo +[*Where the total number of instances comes from*](#) for the call +chain (and matching values across all four state arrays: +`ccpp_suite_state`, `ccpp_suite_data`, `ccpp_group_state`, +`ccpp_model_constituents_obj`). + +If the host doesn't declare `number_of_instances`, every per-instance +allocation falls back to `1` and the framework runs single-instance. + +### Two host-side lifecycle patterns + +Both work; pick whichever fits your model. + +**Pattern A: all instances registered first** +``` +do isuite = 1, num_suites + do iinst = 1, num_instances + call ccpp_register(suite_names(isuite), errmsg, errflg, iinst) + end do +end do +do iinst = 1, num_instances + call ccpp_register_constituents(host_constituents, iinst, errflg, errmsg) + call ccpp_initialize_constituents(ncols, num_layers, iinst, errflg, errmsg) +end do +do isuite = 1, num_suites + do iinst = 1, num_instances + call ccpp_init(suite_names(isuite), errmsg, errflg, iinst) + end do +end do +! ... time-stepping ... +do iinst = 1, num_instances + call ccpp_deallocate_dynamic_constituents(iinst) + ... +end do +``` + +**Pattern B: serial per instance** +``` +do iinst = 1, num_instances + do isuite = 1, num_suites + call ccpp_register(suite_names(isuite), errmsg, errflg, iinst) + end do + call ccpp_register_constituents(host_constituents, iinst, errflg, errmsg) + call ccpp_initialize_constituents(ncols, num_layers, iinst, errflg, errmsg) + do isuite = 1, num_suites + call ccpp_init(suite_names(isuite), errmsg, errflg, iinst) + end do + ! ... per-instance time-stepping ... + call ccpp_deallocate_dynamic_constituents(iinst) +end do +``` + +### Last-to-leave teardown + +`ccpp_deallocate_dynamic_constituents(inst)`: +1. Per-instance `obj(inst)%reset()`. +2. Iterates every `obj(i)`; if any has `%const_props_locked() == .true.`, + returns early. +3. Otherwise (every instance reset): deallocates the shared per-suite + buffers, deallocates `ccpp_model_constituents_obj(:)`, and zeros every + `index_of_` integer. + +This works for both lifecycle patterns above. + +--- + +## 8. Limitations and gotchas + +> **Note (2026-05-12).** Several items in this section are under active +> discussion for an upcoming framework + generator overhaul. See +> `doc/constituents_overhaul.md` for the full architectural review and +> three reform proposals. + +### Framework property ownership (2026-05-12) + +The framework's `ccpp_constituent_properties_t` now carries a private +`framework_owns_me` flag (default `.false.`) with +`is_framework_owned()` getter and `set_framework_owned(value)` setter. +`ccpt_deallocate` only deallocates the underlying prop when the flag +is `.true.`; otherwise it just nullifies its pointer. + +Under capgen's explicit-registration model, all +`ccpp_constituent_properties_t` objects are **target-owned by the +caller** (the host's `host_constituents(:)` array, or the per-suite +`_dynamic_constituents(:)` buffer). We never set the flag, so +the framework correctly skips deallocation. Hosts that hand-allocate +property objects on the heap and want the framework to free them must +call `set_framework_owned(.true.)` before passing to `%new_field`. + +### Missing setters (framework gap) + +The framework lacks setters for `advected`, `diagnostic_name`, +`default_value` (and `mixing_ratio_type`). This means once a +constituent is `%instantiate`d, those properties cannot be changed. +If your host needs to override a scheme-supplied `diagnostic_name` or +`advected` value, you currently cannot — open item in the constituents +overhaul proposal. + +### `ccpp_scheme_utils` singleton + +`ccpp_scheme_utils.ccpp_initialize_constituent_ptr` accepts only one +singleton pointer. It's a framework-level convenience used by cam-sima +schemes that call `ccpp_constituent_index` from `ccpp_scheme_utils`. + +`ccpp_initialize_constituents` calls +`ccpp_initialize_constituent_ptr(obj(inst))` on each instance, but +**only the first call across instances actually sets the pointer** +(the routine is internally guarded by an `initialized` flag). +Subsequent calls are silent no-ops. + +For multi-instance hosts, schemes that use +`ccpp_scheme_utils.ccpp_constituent_index` will see only the first +instance's object — a known limitation inherited from the framework +module's design. Schemes that use the per-instance accessors +(`obj(inst)%const_index(...)` via `ccpp_const_get_index`) are +unaffected. + +### Constituent metadata is identical across instances + +The constituent table (which constituents exist, their properties, the +`index_of_` mapping) is **identical** for every instance. Every +instance's `obj(i)` has the same hash table, populated identically by +its own `ccpp_register_constituents` call. + +This means: + +- `ccpp_number_constituents` returns the same value regardless of + `instance_number`. +- `ccpp_const_get_index` returns the same index regardless of + `instance_number`. +- The `index_of_` integers are populated identically by every + instance's `ccpp_initialize_constituents` (last-write-wins is fine + since every write is the same value). + +`instance_number` is still in the signatures of these routines — see +[§5](#5-public-api-reference) for the rationale. + +### Forbidden patterns recap + +This is rejected at code-gen time (Rule 4 of [§2](#2-the-four-rules-scheme-author-conventions)): + +- `is_constituent + intent=out + non-tendency std_name` — physics phases + may only produce tendencies, not new base constituents. + +(As of 2026-06-05, `intent=in/inout + tendency_of_*` is **allowed** — a +constituent tendency may be *consumed*, resolving to `%vars_layer_tend`. +Only *producing* a tendency uses `intent=out`.) + +### Subscript indices in sliced local_names must be standard names + +If a host metadata variable is declared with a sliced local name +like `q(:,:,index_of_water_vapor_specific_humidity)`, every subscript +token (other than `:` and integer literals) must be a known standard +name. Otherwise the resolver raises a `CCPPError` with a clear +message naming the offending token. + +### Open work items + +- **Unconditional `ccpp_host_constituents.F90` emission.** The + generator currently emits `ccpp_host_constituents.F90` for every + build, even when no scheme or host actually uses the constituent + system (no `ccpp_constituent_properties_t(:)` register-phase arg, + no `is_constituent`-flagged scheme arg, no framework-named + `index_of_` / `ccpp_constituents` / etc. claimed by capgen). + When the host owns its own indices (SCM/GFS) and no scheme exercises + the constituent path, the generated file is dead code that should be + suppressed. Tracked as a deferred item; the `host_dict` precedence + rule above already keeps the file *correct* (empty) in that case. + +--- + +## 9. Differences from original capgen + +| Aspect | Original capgen | capgen | +|---|---|---| +| Constituent object location | Generated `_ccpp_cap.F90` module | `ccpp_host_constituents.F90` (one per generator run) | +| Per-instance | No (single instance) | Yes (`obj(:)` allocatable, sized to `number_of_instances`) | +| Routine name prefix | `_ccpp_register_constituents`, etc. | `ccpp_register_constituents`, etc. (no host prefix; one set per generator run) | +| Routine signatures | No `instance_number` arg | `instance_number` in every per-instance routine | +| Host metadata for constituent obj | None (auto-created by generator) | None (still auto-created by generator) | +| Module-level pointers | `_constituents_array` etc. as functions returning pointers | Same idea, now per-instance via `instance_number` arg | +| Scheme std-name set | `_model_const_stdnames` parameter array | `ccpp_model_const_stdnames` parameter array (no host prefix) | +| Host-facing API surface | `_ccpp_register_constituents`, `_ccpp_initialize_constituents`, `_ccpp_number_constituents`, `_ccpp_is_scheme_constituent`, `_ccpp_gather_constituents`, `_ccpp_update_constituents`, `_ccpp_deallocate_dynamic_constituents`, `_constituents_array`, `_advected_constituents_array`, `_model_const_properties`, `_const_get_index` | Same surface, no `_` prefix | +| Dynamic constituent buffer dimensionality | 1D, per host | 1D **per instance** (wrapper-DDT array indexed by `instance_number`; was shared across instances pre-2026-05-18, until the combined multi-instance + constituents e2e test surfaced a latent set_const_index conflict) | +| Static suite constituents | Auto-cloned by `ConstituentVarDict.find_variable` and registered via `_constituents_copy_const` accessors | Default behaviour: tracked at code-gen time via `is_constituent` flag; included in the constituent table only if a register-phase scheme produces them (rule 1). Schemes that *consume* a base constituent (rule 2) don't trigger registration — the constituent must be registered by SOMEONE (host or another scheme's register) for the access to work at runtime. **Opt-in shim** `--legacy-auto-clone-constituents` (2026-05-21, transient) reinstates original capgen's auto-clone path for legacy hosts; single-instance only. See `doc/auto_clone_constituents.md`. | + +### Migration notes for cam-sima hosts + +- **Scheme metadata**: no changes needed for the 4 schemes that + register constituents via `ccpp_constituent_properties_t` (rule 1) — + those work unchanged. For the ~16 schemes that rely on original + capgen's auto-clone path (`advected = True` on a `_run` arg with no + matching register-phase source), pass + `--legacy-auto-clone-constituents` to `ccpp_capgen.py` and + `ccpp_validator.py` — capgen then auto-registers those + constituents into the per-suite dynamic-constituents buffer the same + way original capgen did. See `doc/auto_clone_constituents.md`. +- **Host metadata**: drop any explicit declaration of + `ccpp_model_constituents_object` if you carried one over from a + previous capgen experiment — the generator owns it now. +- **Host Fortran**: change all `_ccpp_*_constituents` calls to + the unprefixed names (`ccpp_register_constituents` etc.) and add + `instance_number` to every call site. + +--- + +## 10. Worked example + +A minimal cam-sima-style suite with one scheme that consumes a base +constituent and produces its tendency. + +### Scheme metadata (`consume_constituent.meta`) + +``` +[ccpp-table-properties] + name = consume_constituent + type = scheme + +[ccpp-arg-table] + name = consume_constituent_run + type = scheme +[ cldliq ] + standard_name = cloud_liquid_water_mixing_ratio + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = in + advected = .true. +[ tend_cldliq ] + standard_name = tendency_of_cloud_liquid_water_mixing_ratio + units = kg kg-1 s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = out + constituent = .true. +[ errmsg ] + ... +[ errflg ] + ... +``` + +### Host metadata (`my_host.meta`) + +``` +[ccpp-table-properties] + name = my_host + type = host + +[ccpp-arg-table] + name = my_host + type = host +[ ncols ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + units = count + dimensions = () + type = integer +[ ninstances ] + standard_name = number_of_instances + units = count + dimensions = () + type = integer +``` + +(Plus a `type=control` table declaring `instance_number`, +`horizontal_loop_begin`, `horizontal_loop_end`, `ccpp_error_message`, +`ccpp_error_code`, etc.) + +### Suite XML (`my_suite.xml`) + +```xml + + + + consume_constituent + + +``` + +### Generated `ccpp_host_constituents.F90` (excerpt) + +```fortran +module ccpp_host_constituents + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: & + ccpp_model_constituents_t, ccpp_constituent_properties_t, & + ccpp_constituent_prop_ptr_t + + implicit none + private + + public :: ccpp_model_constituents_obj + public :: index_of_cloud_liquid_water_mixing_ratio + public :: ccpp_register_constituents, ccpp_initialize_constituents + public :: ccpp_is_scheme_constituent, ccpp_number_constituents + public :: ccpp_gather_constituents, ccpp_update_constituents + public :: ccpp_const_get_index, ccpp_constituents_array + public :: ccpp_advected_constituents_array, ccpp_model_const_properties + public :: ccpp_deallocate_dynamic_constituents + public :: ccpp_model_const_stdnames + + type(ccpp_model_constituents_t), target, allocatable :: ccpp_model_constituents_obj(:) + integer :: index_of_cloud_liquid_water_mixing_ratio = 0 + character(len=31), parameter :: ccpp_model_const_stdnames(1) = (/ & + 'cloud_liquid_water_mixing_ratio' /) + +contains + ! ... full subroutine bodies as in §5 ... +end module ccpp_host_constituents +``` + +### Host code skeleton (single-instance illustration) + +```fortran +subroutine my_host_run() + ! my_host_ccpp_cap is the per-host static API module + ! (filename and module name derived from --host-name). + use my_host_ccpp_cap, only: ccpp_register, ccpp_register_constituents, & + ccpp_initialize_constituents, ccpp_init, & + ccpp_physics_run, ccpp_final, & + ccpp_deallocate_dynamic_constituents + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + type(ccpp_constituent_properties_t), allocatable :: host_consts(:) + integer :: errflg + character(len=512) :: errmsg + integer, parameter :: inst = 1 + + ! 1. Run register phase: populates per-suite dynamic-constituent buffers. + call ccpp_register('my_suite', errmsg, errflg, inst) + + ! 2. Build host's own constituent declarations (water vapor, etc.). + allocate(host_consts(1)) + call host_consts(1)%instantiate( & + std_name='water_vapor_specific_humidity', long_name='water vapor', & + units='kg kg-1', vertical_dim='vertical_layer_dimension', & + advected=.true., errcode=errflg, errmsg=errmsg) + + ! 3. Merge host + suite-side constituents into obj(inst). + call ccpp_register_constituents(host_consts, inst, errflg, errmsg) + + ! 4. Allocate vars_layer + bind cached indices. + call ccpp_initialize_constituents(ncols, nlev, inst, errflg, errmsg) + + ! 5. Framework init phase. + call ccpp_init('my_suite', errmsg, errflg, inst) + + ! 6. Time-stepping (omitted). + call ccpp_physics_run('my_suite', 'phys', col_start, col_end, & + thread_num, nthreads, nphys_threads, & + errflg, errmsg, inst) + + ! 7. Shutdown. + call ccpp_final('my_suite', errmsg, errflg, inst) + call ccpp_deallocate_dynamic_constituents(inst) + deallocate(host_consts) +end subroutine my_host_run +``` + +For multi-instance, wrap each per-instance call in +`do iinst = 1, ninstances ... end do` per the patterns in +[§7](#7-multi-instance-design). diff --git a/doc/constituents_overhaul.md b/doc/constituents_overhaul.md new file mode 100644 index 00000000..6343c6b7 --- /dev/null +++ b/doc/constituents_overhaul.md @@ -0,0 +1,1151 @@ +# CCPP Constituents — Architecture Review & Overhaul Discussion + +**Authors:** Dom Heinzeller (lead), Claude (assistant) +**Date drafted:** 2026-05-12 +**Last revised:** 2026-06-05 +**Intended audience:** CCPP framework team, CAM-SIMA team +**Status:** Discussion document — no decisions are final. Proposals +A/B/C below remain pending the upcoming meeting; the bug fix from +Proposal A (the `ccpt_deallocate` ownership flag) and the capgen +internal cleanup from Proposal B (§4.8) have landed; the missing +setters from Proposal A and the `is_match` relaxation from Proposal B +have not. Independent of A/B/C, the per-suite dynamic_constituents +buffer was made per-instance on 2026-05-18 to fix a multi-instance +mutation conflict — see §4.13. Since 2026-06-03 capgen drives the +real CAM-SIMA build (via the `cime_config/capgen_compat/` facade): the +`kessler`, `rrtmgp`, and `se_cslam`/CSLAM (FCAM7 `cam7`) cases all build +and run on Derecho. That integration added the **rule-b** consumer path +(read a constituent or `tendency_of_` without re-flagging — §2.2.3) +and surfaced the host-adapter fix in §4.15; neither changes the A/B/C +decision surface. + +--- + +## Executive summary + +CCPP's "constituent" mechanism — how schemes declare and how the framework +manages tracer species like water vapor, cloud liquid, prescribed ozone, +etc. — has grown organically over the last few years. The result works, +but it carries: + +- **A latent framework bug** in `ccpp_constituent_prop_mod` that crashes on + teardown of explicitly-registered (target-passed) constituent property + arrays. Fixed in capgen's framework copy 2026-05-12; needs to land + upstream. +- **Architectural confusion** about which properties are *physics-portable* + (the scheme owns them) versus *host-configuration* (the host owns them). + Today schemes are forced to supply host-specific values (`diag_name` is + the worst offender) at `%instantiate` time. +- **Setter API gaps**: properties that the host wants to override after + scheme-side registration (`advected`, `diagnostic_name`, `default_value`) + have no setters; `is_match` is overly strict about properties hosts + should be free to change. +- **Two registration models** coexist — original capgen's auto-clone of + is_constituent scheme args, and capgen's/capgen's explicit register-phase + + host-side declaration. Capgen deliberately dropped auto-clone. + +This document is a structured brief for a discussion this week. It does +NOT pre-commit to any decision; it lays out what exists, what's broken, +what we audited, and what proposals are on the table. + +--- + +## Table of contents + +1. [How original capgen handles constituents](#1-how-original-capgen-handles-constituents) +2. [How capgen handles constituents](#2-how-capgen-handles-constituents) +3. [What CAM-SIMA actually needs (audit)](#3-what-cam-sima-actually-needs-audit) +4. [Bugs and design flaws](#4-bugs-and-design-flaws) +5. [Property classification (Class A vs Class B)](#5-property-classification-class-a-vs-class-b) +6. [What to remove, replace, improve](#6-what-to-remove-replace-improve) +7. [Open design questions](#7-open-design-questions) +8. [Three proposals — minimal / clean / deep](#8-three-proposals--minimal--clean--deep) +9. [Appendix: framework setter inventory](#9-appendix-framework-setter-inventory) + +--- + +## 1. How original capgen handles constituents + +### 1.1 Mental model + +Original capgen treats constituents as a **separate scope** between +suite and host: + +``` +group → suite → ConstituentVarDict → host +``` + +A scheme arg flagged `constituent = True` in metadata is matched first +against group/suite/ConstituentVarDict, and only against host as a last +resort. The ConstituentVarDict is a synthetic dictionary whose entries +are auto-created by `find_variable()` when a scheme metadata declares a +constituent dependency. + +### 1.2 Auto-clone of `is_constituent` scheme args + +Every scheme arg with non-default `advected`, `constituent`, or +`molar_mass` is treated as a *registration*. The generator emits, into +the host cap, a routine `_constituents_ccpp_create_constituent_array` +that: + +1. Allocates a `ccpp_constituent_properties_t` pointer per scheme arg. +2. Calls `%instantiate(...)` populating fields **from the scheme + metadata directly** — `std_name`, `long_name`, `diagnostic_name`, + `units`, `default_value`, `advected`, `vertical_dim`, etc. (See + `scripts/constituents.py:565`.) +3. Adds it to the model constituents object via `%new_field`. + +After this auto-clone runs, the host's hand-written +`host_constituents(:)` array is appended, then `%lock_table` finalizes +the hash table. + +### 1.3 The host-cap-owned `ccpp_model_constituents_obj` + +Original capgen generates **one** `ccpp_model_constituents_obj` per +generator invocation, declared module-level in `_ccpp_cap.F90`. +Single global; not per-instance. (CAM-SIMA runs one host per +executable, so single-instance is fine for them.) + +### 1.4 Scheme-side `%instantiate` registration (the other path) + +A scheme may also register constituents via a register-phase argument: + +```fortran +type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) +``` + +The scheme allocates the array, calls `%instantiate` per entry, and +returns it. Original capgen wires this through a per-suite +"dynamic constituents" buffer and merges it during host-cap setup, +alongside the auto-cloned set. + +So original capgen really supports **three** registration sources: + +- Host: hand-written `host_constituents(:)` arg. +- Suite-dynamic: register-phase scheme args. +- Suite-static: auto-cloned from any `is_constituent` consumer. + +All three flow into one `%new_field` table. + +### 1.5 Lifecycle + +- `_ccpp_register_constituents(host_constituents, ...)` runs the + three-source merge. +- `ccpp_initialize_constituent_ptr(const_obj, ...)` (in + `ccpp_scheme_utils`) caches a pointer for the `ccpp_constituent_index` + lookup. +- Phase entry points access `vars_layer` / `vars_layer_tend` via cached + `index_of_` integers. + +### 1.6 What's good about original capgen's approach + +- Schemes declare a constituent dependency once in metadata; no manual + Fortran registration ever needed for "static" tracers. +- Host doesn't have to enumerate every species every scheme wants. +- Works for CAM-SIMA's current scheme catalog. + +### 1.7 What's painful about original capgen's approach + +- The auto-clone path is **invisible** to anyone reading the scheme + Fortran — the registration happens in generated code. +- `ConstituentVarDict` is a synthetic scope, conceptually subtle, and + doesn't generalize cleanly to multi-instance. +- The auto-clone path lifts `diagnostic_name` and `default_value` from + scheme metadata, but those values are often host-specific (see §4.4). +- Three sources of registration with overlap mean two registrations of + the same `std_name` may collide; original capgen relies on + `is_match` (units, advected, thermo_active, water_species) to dedup, + which means schemes accidentally diverge on `advected` and trip the + "incompatible constituent" error. + +--- + +## 2. How capgen handles constituents + +### 2.1 Mental model + +No synthetic scope. Constituents are *one of four* sources for any +scheme arg: + +``` +control | host | suite | constituent +``` + +The resolver classifies each scheme arg into exactly one source. A +`constituent` source means the value will be accessed at runtime as +`ccpp_model_constituents_obj()%vars_layer(:, :, index_of_)` +(or `%vars_layer_tend(...)` for `tendency_of_` outputs). + +### 2.2 The scheme-author rules + +(See `doc/constituents.md` for full details; this is the summary.) + +1. **Register** — register-phase scheme args of type + `ccpp_constituent_properties_t(:), intent=out, allocatable` declare + new constituents the scheme contributes. +2. **Flag what you own** — a physics-phase arg flagged `advected=true` + (or `molar_mass=...` or `constituent=true`) marks its standard name as + a constituent: a base species read via `%vars_layer` + (`intent=in/inout`), or — when the name is `tendency_of_` — a + constituent tendency *written* via `%vars_layer_tend` (`intent=out`). + A base constituent therefore uses `intent=inout` (read-modify-write the + shared column), never `intent=out`; `intent=out` is reserved for + tendencies. (This is why CAM-SIMA's `state_converters` dry→moist + converters declare the moist mixing ratios `intent=inout`.) +3. **Consume without re-flagging (rule b)** — a scheme that merely READS a + name some *other* scheme flags as a constituent does **not** repeat the + flag. Whether a standard name is a constituent or an ordinary variable + is the **host's** decision (CAM-SIMA exposes water vapor as a + constituent; CCPP-SCM may expose the same name as an ordinary host + variable), so capgen infers it from the scheme-metadata-wide set of + flagged names (`VariableResolver.constituent_stdnames()`) rather than + from the consumer's own metadata. An unflagged `intent=in` read of the + base name resolves to `%vars_layer(...)`; an unflagged `intent=in` read + of `tendency_of_` resolves to `%vars_layer_tend(:, index_of_)` + — the same column a tendency *producer* wrote. **Host/earlier-suite + provision wins**: if the host declares the name, or an earlier scheme + already produced it as an ordinary variable, normal host/suite + resolution takes over. (Worked example: in the CAM-SIMA `cam7` suite + the convection/stratiform schemes write `tendency_of_water_vapor_...` + as a flagged constituent tendency, and the unflagged `sima_diagnostics` + schemes read it back via this rule — see §4.15.) +4. **Mismatched combinations are errors** — a constituent-FLAGGED + `intent=out` arg whose name is not a `tendency_of_*` is a codegen-time + error: physics phases may only PRODUCE tendencies; new base + constituents must be declared in the register phase. + +### 2.3 Two registration sources (no auto-clone) + +- **Host**: hand-written `host_constituents(:)`, passed into + `ccpp_register_constituents(host_constituents, instance_number, ...)`. +- **Suite-dynamic**: register-phase scheme args, accumulated into a + per-suite buffer `_dynamic_constituents(:)` by `_register`, + drained into `ccpp_model_constituents_obj(inst)` by + `ccpp_register_constituents`. + +The auto-clone-from-metadata path is **gone from capgen's default +behaviour**. If a scheme declares `advected=true` on an arg but no +source registers that standard name, capgen emits a runtime check +during `ccpp_initialize_constituents` that errors with the missing +name. + +**Legacy escape hatch** (added 2026-05-21): the opt-in CLI flag +`--legacy-auto-clone-constituents` reinstates the original +auto-clone path for hosts whose scheme metadata predates explicit +registration (production CAM-SIMA's atmospheric_physics tree is the +immediate consumer). This is a transient migration shim — see +`doc/auto_clone_constituents.md` for the full reference and +removal procedure. It is single-instance only and explicitly +flagged so future capgen work is *not* expected to keep it +indefinitely. The reform proposals in §6–§8 below are unchanged by +the shim's existence: capgen's chosen architecture is still +explicit registration. + +### 2.4 Per-instance state + +Everything is per-instance: + +```fortran +type(ccpp_model_constituents_t), allocatable :: ccpp_model_constituents_obj(:) + ! indexed by instance_number +``` + +All host-facing entry points take `instance_number`: + +``` +ccpp_register_constituents (host_constituents, instance_number, errflg, errmsg) +ccpp_initialize_constituents (ncols, num_layers, instance_number, errflg, errmsg) +ccpp_number_constituents (num_flds, advected, instance_number, errflg, errmsg) +ccpp_gather_constituents (const_array, instance_number, errflg, errmsg) +ccpp_update_constituents (const_array, instance_number, errflg, errmsg) +ccpp_const_get_index (stdname, const_index, instance_number, errflg, errmsg) +ccpp_constituents_array (instance_number) => pointer +ccpp_advected_constituents_array (instance_number) => pointer +ccpp_model_const_properties (instance_number) => pointer +ccpp_deallocate_dynamic_constituents (instance_number, ...) +``` + +`ccpp_is_scheme_constituent(var_name, ...)` and the +`ccpp_model_const_stdnames(:)` parameter array are NOT per-instance — +the standard-name catalog is identical across instances. + +### 2.5 Lifecycle + +``` +ccpp_register(suite_name, instance_number, ...) + └─ _register → packs scheme-dynamic constituents into + _dynamic_constituents(instance)%items + (per-instance wrapper-DDT array; each instance + allocates and fills its own slot — see §4.13) + ↓ +ccpp_register_constituents(host_constituents, instance_number, ...) + └─ initialize_table(num_host_consts + num_suite_consts) + └─ new_field(host_consts ...) + └─ new_field(_dynamic_constituents ...) + └─ lock_table + ↓ +ccpp_initialize_constituents(ncols, num_layers, instance_number, ...) + └─ lock_data (allocates vars_layer, vars_layer_tend, vars_minvalue) + └─ ccpp_initialize_constituent_ptr (first instance wins; documented limit) + └─ %const_index('') for each enumerated constituent + └─ post-lookup int_unassigned check → clear error message + ↓ +ccpp_init(suite_name, instance_number, ...) + └─ _init → binds module-level pointers + ↓ +... physics phases ... + ↓ +ccpp_final(suite_name, instance_number, ...) + └─ _final → nullifies + last-to-leave deallocates + ↓ +ccpp_deallocate_dynamic_constituents(instance_number, ...) + └─ ccp_model_constituents_obj(inst)%reset + ↓ (in _final, last-to-leave) + deallocate(_dynamic_constituents) +``` + +### 2.6 What's good + +- Explicit. Every constituent registration is visible in someone's + Fortran source. +- Multi-instance from day one. +- The "four rules" are small enough to fit on a slide. +- Resolver-time + codegen-time + runtime checks catch the most common + mistakes. + +### 2.7 What's still painful + +Covered in §4. + +--- + +## 3. What CAM-SIMA actually needs (audit) + +### 3.1 Scheme-side registration usage + +We audited `EXT/cam-sima/atmospheric_physics/schemes/` for use of the +register-phase `ccpp_constituent_properties_t(:)` pattern: + +| Scheme | File | Registers | +|---|---|---| +| RRTMGP constituents | `schemes/rrtmgp/rrtmgp_constituents.meta` | radiative-active species | +| MUSICA chemistry | `schemes/musica/musica_ccpp.meta` | chemical species from MUSICA | +| Prescribed aerosols | `schemes/chemistry/prescribed_aerosols.meta` | aerosol species | +| Prescribed ozone | `schemes/chemistry/prescribed_ozone.meta` | ozone | + +**Total: 4 of 128 schemes** in the atmospheric_physics tree use +scheme-side registration. The other 124 only **consume** constituents +(`advected=true` + `intent=in/inout` in metadata, accessed via the +framework's `vars_layer`). + +This is a small enough number that an alternative "host-only +registration" model is feasible: move those 4 register calls into the +host (or into helper modules the host calls), and the rest of the +catalog only consumes. + +### 3.2 Host-side patterns + +`EXT/cam-sima/CAM-SIMA/src/physics/utils/cam_constituents.F90` wraps +the framework setters and exposes: + +- `const_set_thermo_active(const_obj | const_ind, value)` +- `const_set_water_species(const_obj | const_ind, value)` +- `const_set_minimum(...)` + +CAM-SIMA actively **calls these setters at runtime** — schemes don't +supply `thermo_active` at instantiate time; the host configures it +afterwards. This is direct evidence that the "post-instantiation +override" pattern is real and used today, and that the framework's +setter API is load-bearing. + +### 3.3 What CAM-SIMA does **not** do + +- It does not rely on auto-clone for `diag_name`. The scheme-side + register calls in the 4 schemes do supply `diag_name`, but those + values are CAM-SIMA's; a different host would need different ones. +- It does not use `ccpp_constituent_index` (the + `ccpp_scheme_utils`-singleton-based lookup) extensively — most + access goes through the framework's `index_of_` integers. + +### 3.4 What CAM-SIMA's host-cap-owned constituent object looks like + +Because original capgen generates **one** `ccpp_model_constituents_obj` +per generator invocation, and CAM-SIMA uses one generator invocation per +executable, CAM-SIMA effectively runs single-instance today. A +multi-instance CAM-SIMA (sub-columns, ensembles) would expose the +single-global limitation immediately. + +--- + +## 4. Bugs and design flaws + +This section lists known issues across the three layers (framework, +original capgen, capgen). Items marked **(FIXED)** were resolved +2026-05-12 and either are or will be PRs; items marked **(OPEN)** are +intentionally left for this discussion. + +### 4.1 Framework: `ccpt_deallocate` ownership bug (FIXED in capgen tree, needs upstream PR) + +- **Location**: `src/ccpp_constituent_prop_mod.F90`, `ccpt_deallocate` + + `ccpt_set`. +- **Symptom**: `free(): invalid size` crash when + `ccp_model_const_reset` is called on a properly-locked table whose + entries came from pointer-assigned targets (the common pattern + under capgen's explicit registration; also potentially under + original capgen's `host_constituents` path). +- **Root cause**: `ccpt_set` does pointer assignment (`this%prop => + const_ptr`); `ccpt_deallocate` does an unconditional + `deallocate(this%prop)`. The deallocate is correct only when the + caller allocated `const_ptr` on the heap and transferred ownership. +- **Why it didn't surface earlier**: original capgen's advection test + only calls `deallocate` once between a *failing* register and a + *successful* one — at that point `lock_table` has not populated + `const_metadata`, so the broken inner loop is skipped. Capgen + triggers it because its teardown calls `reset` after a successful + lock. +- **Fix landed 2026-05-12**: added `framework_owns_me` private flag on + `ccpp_constituent_properties_t` (default `.false.`) with + `is_framework_owned()` getter and `set_framework_owned(value)` + setter; `ccpt_deallocate` now only deallocates when the flag is set. + Original capgen's auto-clone path in `scripts/constituents.py` + updated to call `set_framework_owned(.true.)` after `allocate`. + Diffs in `src/ccpp_constituent_prop_mod.F90` (and capgen's + parallel copy) + `scripts/constituents.py`. +- **Status**: framework tests pass; capgen unit-test suite (1127 passing + as of 2026-05-13) is green. Still needs upstream PR to ccpp-framework + + original ccpp-capgen. + +### 4.2 Framework: missing setters (OPEN) + +| Property | Optional in `%instantiate`? | Has setter? | `is_match`-checked? | +|---|---|---|---| +| `std_name` | required | — | (lookup key) | +| `long_name` | required | — | no | +| `diag_name` | required | **NO** | no | +| `units` | required | — | **yes** | +| `vertical_dim` | required | — | no | +| `advected` | optional (default .false.) | **NO** | **yes** | +| `default_value` | optional | **NO** | no | +| `min_value` | optional | `set_minimum` | no | +| `molar_mass` | optional | `set_molar_mass` | no | +| `water_species` | optional (default .false.) | `set_water_species` | **yes** | +| `mixing_ratio_type`| optional | **NO** | no | +| `thermo_active` | not in instantiate | `set_thermo_active` | **yes** | +| `const_index` | internal | `set_const_index` | no | + +**Pain points**: + +- `advected` is `is_match`-checked AND has no setter. Once registered, + immutable. If a scheme and the host disagree, you get the + "incompatible constituent" error and you cannot reconcile from + Fortran. +- `diag_name` is required (cannot be omitted at instantiate) AND has + no setter. A scheme must pick a value at registration time; that + value is then frozen. +- `default_value` is silently optional. If omitted, the constituent + array initializes to `huge(real)` and downstream comparisons fail + in surprising ways (we burnt half a day on this 2026-05-12). +- `thermo_active` is the only property in the "post-instantiate-only" + shape: it has a setter but isn't a `%instantiate` arg. The + asymmetry is confusing. + +### 4.3 Framework: `is_match` is too strict (OPEN) + +`is_match` (in `ccp_is_match`) checks `units`, `advected`, +`thermo_active`, `water_species`. Three of those four (`advected`, +`thermo_active`, `water_species`) are properties the host legitimately +overrides post-registration. Two registrations of the same `std_name` +with the same `units` but different `advected` should be a +duplicate-dedup (host wins), not a hard error. + +### 4.4 Framework: `diag_name` portability problem (OPEN) + +Diagnostic output names are host-specific. CAM-SIMA names cloud +liquid mixing ratio `CLDLIQ`; UFS would call it something else. Yet +`%instantiate` makes `diag_name` a *required* arg, forcing schemes to +either: + +- Pick a host-specific value (couples the scheme to a host), or +- Pick a "neutral" default that no host's diagnostic tooling + recognizes. + +The current de-facto pattern in CAM-SIMA scheme code is to pick a +CAM-SIMA-flavoured value and ship it. Any port to UFS would need to +either monkey-patch or fork the scheme. + +A clean fix: +1. Make `diag_name` optional at `%instantiate` (default to empty + string or `std_name`). +2. Add `set_diagnostic_name(value)` setter. +3. Host overrides per-registration after `ccpp_register_constituents`. + +### 4.5 Original capgen: implicit registration (OPEN — observation) + +The auto-clone path is generator magic. Reading scheme metadata +doesn't tell you whether the scheme's args result in registration; you +have to know that `advected=true` triggers it. This is a documentation ++ comprehension problem more than a bug. + +### 4.6 Original capgen: single-instance `ccpp_model_constituents_obj` (OPEN — limitation) + +The host cap declares one global. Multi-instance hosts would need to +either generate one cap per instance or restructure. + +### 4.7 Original capgen: `ConstituentVarDict` complexity (OPEN — observation) + +The synthetic scope between suite and host serves correctness but +adds a code path that most contributors don't read. If we drop it +(capgen has), the variable-matching algorithm shrinks. + +### 4.8 Capgen: `_FRAMEWORK_CONST_DIM_INPUTS` cleanup (LANDED 2026-05-13) + +`generator/host_cap.py` no longer carries the hand-curated frozenset of +standard names; framework-constituent dimension references now ride on a +dedicated `used_const_dim_std_names` field on `ResolvedArg`. Closes the +"hand-curated → structured field" REVISIT note that was in the code. + +### 4.9 Capgen: no codegen-time cross-check of scheme registration (OPEN) + +The resolver knows every `is_constituent` arg's standard name (in +`SuiteResolution.constituent_index_names`) but doesn't know what each +scheme's `_register` subroutine actually `%instantiate`s. Today's +guarantee is a runtime check (the `int_unassigned` validation we +added 2026-05-12). Stronger options: + +- (a) New metadata attribute `registers_std_names = a, b, c` on + register-phase tables; codegen errors at generation time. +- (b) Parse scheme `_register` Fortran for `%instantiate(std_name=…)` + calls and cross-check. +- (c) Keep runtime check as authoritative, document the gap. + +### 4.10 Capgen: scheme-metadata `diagnostic_name` for is_constituent args is host-specific (OPEN) + +Same issue as §4.4 but in capgen's metadata layer. Today's +`diagnostic_name` attribute on a scheme metadata arg flows into +`datatable.xml` and is then trusted as "the" diagnostic name. If we +adopt setter-based class-B overrides, this attribute should either be +dropped for constituent args or marked as a default-only hint. + +### 4.11 Capgen: `ccpp_scheme_utils` singleton (OPEN — documented limit) + +`ccpp_initialize_constituent_ptr(const_obj)` stores a single module-level +pointer. Schemes that use `ccpp_constituent_index(stdname)` get that +pointer back. Under multi-instance, only the first instance's +pointer is retained — `ccpp_constituent_index` queries from +within a scheme will always reflect instance 1. CAM-SIMA's 4 +scheme-registering schemes don't rely on this; documented in +`doc/constituents.md` §8. Real fix requires either threading +`instance_number` through `ccpp_constituent_index` (interface +change) or maintaining a per-instance pointer table. + +### 4.12 Capgen: drop `diagnostic_name_fixed`, keep only `diagnostic_name` (OPEN — proposed simplification) + +Today the metadata layer carries two mutually-exclusive scheme-arg +attributes: + +- `diagnostic_name = X` — emits `diagnostic_name="X"` in `datatable.xml`; + defaults to `local_name` when absent. +- `diagnostic_name_fixed = Y` — emits `diagnostic_name_fixed="Y"` in + `datatable.xml`; the `diagnostic_name` slot stays empty (no + auto-default to `local_name`). + +The behavioral difference is purely *which attribute name* host +tooling sees in `datatable.xml` — both attributes carry the same kind +of value (a Fortran-identifier-shaped string), and both are passed +through unmodified. `_fixed` is a signal to the host "use verbatim, do +not decorate or transform"; but `diagnostic_name = X` already means +exactly that — the cap code never decorates the value, and any host +tooling that wants to decorate would have to opt in by parsing a +separate attribute (or by syntactic convention on the value itself). + +**Proposal:** Remove `diagnostic_name_fixed` from the metadata layer +and the parser. Keep `diagnostic_name` with the existing defaulting +rule (explicit → use it; absent → fall back to `local_name`). Hosts +that today rely on the `_fixed` semantic ("don't auto-default to +`local_name`") get the same outcome by simply *setting* +`diagnostic_name` to the desired exact value. + +Touchpoints to retire: + +- `metadata/parse_tools/parse_checkers.py::check_diagnostic_fixed` and + the mutual-exclusion block at the top of `check_diagnostic_id`. +- `metadata_table.py::MetaVar._KNOWN_ATTRS` entry and the + `@property diagnostic_name` fallback that returns `''` when + `_diagnostic_name_fixed` is set. +- `generator/datatable.py:267-269` emission of the + `diagnostic_name_fixed` XML attribute. +- Existing unit-test coverage for `diagnostic_name_fixed` becomes + obsolete and is removed (not migrated). + +**Why it's worth doing as part of the overhaul:** the attribute has no +unique semantics that `diagnostic_name` can't express, and dropping it +shrinks the metadata-layer surface area at the same time the +`set_diagnostic_name(value)` framework setter (§4.4 / §4.10) is being +added on the framework side. Hosts that want runtime override get +`set_diagnostic_name`; hosts that want metadata-declared values get +`diagnostic_name`. There is no third use case that needs `_fixed`. + +**Risk:** non-CCPP-ng metadata in the wild may carry +`diagnostic_name_fixed`. Mitigation: a one-line legacy-mode rewrite +(`metadata/legacy_compat.py`) translates the deprecated attribute to +`diagnostic_name` at parse time with a loud warning, identical in +spirit to the existing `horizontal_loop_extent → horizontal_dimension` +shim. Remove the rewrite once known consumers are migrated. + +### 4.13 Capgen: per-suite `dynamic_constituents` buffer was shared across instances (FIXED 2026-05-18) + +- **Location**: `capgen/generator/host_constituents.py` (buffer + declaration + `ccpp_register_constituents` iteration); + `capgen/generator/suite_cap.py::_register_lines` (the two-pass + count→allocate→pack inside `_register`). +- **Symptom**: with two or more instances and any register-phase + scheme that produces constituents, the second per-instance + `ccpp_register_constituents` call fails with `ccp_set_const_index + ccpp_constituent_properties_t const index is already set`. +- **Root cause**: the per-suite buffer + `_dynamic_constituents(:)` was declared as a single shared + 1-D array of `ccpp_constituent_properties_t`, filled exactly once on + first instance entry (`.not. allocated(buf)` gate). + `ccpp_register_constituents` then iterates that shared buffer per + instance and calls `%new_field(const_prop)` on each property + object. `%new_field` calls `ccp_set_const_index`, which **mutates + the property object** by writing `const_ind`. Instance 1 set + `const_ind` on every shared object; instance 2's call tripped the + "set exactly once" guard. +- **Latent companion bug**: the same shared-mutation pattern means + that once Proposal B's class-B setters (`set_advected`, + `set_diagnostic_name`, `set_water_species` per-instance, etc.) are + exercised, instance 1's setter call would silently corrupt instance + 2's view of the property. No "already set" guard exists on those + setters today. +- **Why it didn't surface earlier**: the advection end-to-end test is + single-instance; the instances end-to-end test has no constituents. + Surfaced by the new `instances_advection` combined test + (`end-to-end-tests/instances_advection/`) on first run. +- **Fix landed 2026-05-18**: the per-suite buffer is now a wrapper-DDT + array indexed by `instance_number`: + ```fortran + type :: ccpp_dyn_const_buffer_t + type(ccpp_constituent_properties_t), allocatable :: items(:) + end type + type(ccpp_dyn_const_buffer_t), allocatable, target :: _dynamic_constituents(:) + ``` + The outer array is allocated to `number_of_instances` on first call; + each instance independently runs the two-pass count+pack into its + own `%items` slot. `ccpp_register_constituents` iterates + `_dynamic_constituents(instance)%items` so each instance's + `new_field` calls operate on **distinct** property objects. + Scheme `_register` routines are now called N times instead of once + (negligible cost — typical register bodies are a few `%instantiate` + calls), in exchange for clean per-instance isolation. +- **Cost**: ~50 lines across the two generator emitters, plus updates + to six pinned unit tests. No CAM-SIMA / NEPTUNE / SCM coordination + needed (host-facing API unchanged). +- **Status**: framework tests pass; full unit-test suite + (1319 tests at fix landing, 1426 as of 2026-06-01) is green; all 10 + end-to-end tests pass. +- **Position relative to Proposals A/B/C**: orthogonal — none of the + three proposed touching the buffer. Independently adopted. + +### 4.14 Capgen: error-output keyword inconsistency across emitted public API (OPEN — observation) + +- **Location**: `capgen/generator/host_cap.py:370,446,*` (lifecycle + subs) vs `capgen/generator/host_constituents.py` (the entire + constituent wrapper family). +- **Symptom**: the public Fortran argument carrying the CCPP error + flag does not have a consistent name across the cap's surface area. + - **Lifecycle subs** (`ccpp_register`, `ccpp_init`, `ccpp_physics_*`, + `ccpp_final`, plus the five `ccpp_physics_suite_*` introspection + routines) read the host's `ccpp_error_code` control-var + `local_name` from `host_dict` and use *that* as the public arg + name. A host that calls `[errflg]` `errflg` ends up with + `subroutine ccpp_register(suite_name, errflg, errmsg)`; a host + that calls `[errcode]` `errcode` ends up with + `subroutine ccpp_register(suite_name, errcode, errmsg)`. This is + the host-controlled side. + - **Constituent wrappers** (`ccpp_register_constituents`, + `ccpp_initialize_constituents`, `ccpp_is_scheme_constituent`, + `ccpp_number_constituents`, `ccpp_gather_constituents`, + `ccpp_update_constituents`, `ccpp_const_get_index`) hard-code + `errcode` regardless of what the host declared. As of + 2026-06-03 the hard-code is `errcode` (renamed from `errflg` for + consistency with the framework methods on + `ccpp_constituent_properties_t` and `ccpp_model_constituents_t`, + which all expose `errcode=`). Before 2026-06-03 it was `errflg`, + which broke any host whose control-var convention was + `errcode` -- including the CAM-SIMA build. +- **Resulting cross-cutting hazard**: in a host where the + `ccpp_error_code` local name happens to be `errflg`, the caller + writes + ```fortran + call ccpp_register(suite_name, errflg=errflg, errmsg=errmsg) ! host-name keyword + call ccpp_register_constituents(host_consts, errcode=errflg, errmsg=errmsg) ! hardcoded keyword + ``` + Two different keyword names for the same conceptual argument on + adjacent calls. Confusing but compiles; the host's local variable + is bound by name to whichever keyword the callee defines. +- **Why the constituent wrappers are hardcoded**: the wrappers are + thin shims around framework methods + (`ccpp_model_constituents_t%new_field`, `%lock_table`, + `%num_constituents`, etc.) that all take `errcode=` per + `capgen/src/ccpp_constituent_prop_mod.F90`. Hardcoding `errcode` + on the wrapper means the wrapper body just forwards + `errcode=errcode` instead of `errcode=` -- one + less host-dict lookup, but at the cost of breaking the + "host names what they want" contract. +- **Options to resolve**: + - (a) Plumb the host's `ccpp_error_code` local name through + `host_constituents.py` the same way `host_cap.py` does (via + `_ctrl_local(host_dict, 'ccpp_error_code') or 'errcode'`). + The constituent wrappers' public arg then tracks the host's + convention. Adds 1 dictionary lookup per emitted sub; no + other change. + - (b) Standardise the lifecycle subs on `errcode` too, ignoring + the host's `ccpp_error_code` local name. Simpler internally + but breaks every existing host that ships + `[errflg] standard_name = ccpp_error_code`. + - (c) Status quo (the constituent wrappers' `errcode` hardcode): + document it loudly and live with the cross-API split. +- **Status**: currently option (c). The post-rename build of CAM-SIMA + works because CAM-SIMA's caller code uses `errcode=errflg` (passing + its local var `errflg` to the hardcoded keyword `errcode`). Hosts + with the opposite convention (`[errcode] standard_name = + ccpp_error_code` -> lifecycle subs expose `errcode=`, + constituent wrappers also expose `errcode=`) coincidentally see + consistent keywords today; the hazard is invisible for them. +- **Recommended fix**: option (a). Lines up with the lifecycle + emitter's already-established host-driven pattern. Trivial + implementation cost; eliminates the cross-cutting confusion for + any host whose `ccpp_error_code` local name is not `errcode`. + +### 4.15 CAM-SIMA compat layer: `write_init_files` mis-flagged unflagged constituent-tendency consumers (FIXED 2026-06-05) + +- **Location**: `cime_config/capgen_compat/_var_wrapper.py` in CAM-SIMA + — the facade that lets capgen drive CAM-SIMA's *unchanged* + `write_init_files.py` / `cam_autogen.py` — method + `_VarWrapper.from_resolved_arg`. +- **Symptom**: the `se_cslam` (FCAM7 `cam7`) build failed AFTER cap + generation, inside CAM-SIMA's own init-file generator: + `Error: Missing required host variables: + tendency_of_water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water`. +- **Mechanism**: in `cam7` the convection/stratiform schemes (`dadadj`, + `zm_conv_evap`, `rk_stratiform`, `zm_convr`, + `cloud_particle_sedimentation`) write that tendency as a FLAGGED + constituent tendency (`constituent=true intent=out`) → capgen routes + them to `%vars_layer_tend` (`source='constituent'`, NOT recorded in + `suite_vars`) and the name enters `const_stds`. The four + `sima_diagnostics` schemes read it back `intent=in` UNFLAGGED → rule b + (§2.2.3) → `source='constituent'`, but `ResolvedArg.is_constituent` is + taken from the consumer's OWN flag = `False`. The compat wrapper + derived `advected`/`constituent` only from + `is_constituent`/`is_constituent_arg` → both `False`, and + `source='constituent'` was not in its suite-internal drop set, so + `write_init_files.gather_ccpp_req_vars` saw intent=in + not-constituent + + not-in-host-dict → "missing host variable". +- **Fix**: key the wrapper's `advected`/`constituent` on + `arg.source == 'constituent'` (a strict superset of the two old flags). + `write_init_files` skips constituents from BOTH USE-import and the + initial-conditions read — the constituents object supplies them at + runtime — so flagging the tendency consumer is correct, not a mask. + Verified 43/43 `capgen_compat` + 16/16 `test_write_init_files`, then + confirmed by full `se_cslam` runs to completion under both gnu and + intel (bit-comparable results). +- **Takeaway**: `ResolvedArg.is_constituent` answers "did the SCHEME flag + it"; `source == 'constituent'` answers "is this supplied by the + constituents framework". Any host adapter (the CAM-SIMA compat layer + today, any future one) must key constituent handling on the *source*, + because rule-b inferred consumers legitimately carry + `is_constituent == False`. +- **Position relative to Proposals A/B/C**: orthogonal — a host-adapter + bug exposed by rule b, not a framework constituent-model change. + +--- + +## 5. Property classification (Class A vs Class B) + +Proposed in `design_constituents_mutability.md` 2026-05-12. Each +constituent property is conceptually owned by either the scheme +(physics-portable, immutable once instantiated) or the host +(host-configuration, mutable post-instantiation). + +### Class A — scheme-intrinsic (immutable) + +| Property | Why class A | +|---|---| +| `std_name` | Identity. Cannot change. | +| `long_name` | Human-readable name of the *species*. Not host-specific. | +| `units` | Physics correctness. `is_match`-checked. | +| `vertical_dim` | Scheme's structural expectation (interface vs layer). | +| `molar_mass` | Physical constant of the species. | +| `default_value` | (Debatable — see §7) Scheme-appropriate initial value. | + +### Class B — host-configuration (mutable post-instantiation) + +| Property | Why class B | +|---|---| +| `advected` | Whether the host's dycore advects this — host decision. | +| `diag_name` | Host-specific diagnostic system name. | +| `thermo_active` | Host model configuration. | +| `min_value` | Host runtime guardrail. | +| `water_species` | (Borderline — see §7) Physical classification but also host-config. | +| `mixing_ratio_type` | (Borderline — see §7) Depends on dycore convention. | + +### Consequences if adopted + +- `is_match` should check **only class A**. Today it checks 3 of 4 + class-B properties. +- Class B properties need setters. Today + `advected`, `diag_name`, (and `mixing_ratio_type` if it stays + class B) have none. +- `%instantiate` can demote class B from "required + optional" to + "all optional with sane defaults" — `diag_name=''`, + `advected=.false.`, etc. Schemes wouldn't need to set them at all. + +--- + +## 6. What to remove, replace, improve + +### Remove (or stop requiring) + +- **Scheme-metadata `diagnostic_name` on is_constituent args** — host + will override. Keep the attribute valid on non-constituent args + (where it's host tooling documentation, no portability issue). +- **`is_match` checks on advected / water_species / thermo_active** — + class B should not block dedup. +- **The `diag_name` requirement at `%instantiate`** — demote to + optional with `''` default. +- **(Not adopting)** Original capgen's auto-clone path. Already gone + in capgen; this discussion does not propose bringing it back. + Listed for completeness because the option is in memory. + +### Replace + +- **`ConstituentVarDict`** as a concept — capgen already runs + without it. If the framework or future generator code references + it, dropping is fine. +- **Single-global `ccpp_model_constituents_obj`** — capgen's + per-instance array is the replacement. Original capgen could be + retrofitted, but the priority depends on whether multi-instance + enters the original capgen's roadmap. + +### Improve + +- **Add the missing setters**: `set_advected`, `set_diagnostic_name`, + `set_default_value` (if `default_value` becomes class B), + `set_mixing_ratio_type` (if class B). +- **Add a convenience routine** like + `ccpp_get_constituent_props_by_std_name(stdname, instance_number, prop_ptr, errflg, errmsg)` + so hosts can lookup a single constituent's property wrapper by + name without iterating. +- **Codegen-time cross-check** of scheme `_register` calls vs + metadata declarations (preferred: §4.9 option (a) — new + `registers_std_names` attr). +- **Document the lifecycle** clearly. `doc/constituents.md` is + ~960 lines; targeted additions for "register-then-override" + workflow once the new setters land. +- **Capgen-internal cleanup** (LANDED 2026-05-13): replaced + `_FRAMEWORK_CONST_DIM_INPUTS` with a `used_const_dim_std_names` + field on `ResolvedArg`. + +--- + +## 7. Open design questions + +These are the calls we need to make in the meeting. + +### Q1. `default_value` — class A or class B? + +- **Class A argument**: the scheme knows what the species + should be initialized to (zero for "starts empty"; small positive + for "starts at background"); the host doesn't typically override. +- **Class B argument**: hosts may want non-default starting values + (chemistry runs with prescribed initial profiles). +- **Today's reality**: framework has no setter, so it's de-facto + class A. The advection-test issue 2026-05-12 surfaced because we + removed the `default_value=0._kind_phys` from cld_liq.F90's + scheme-side register and had no way to put it back; restoring it + in the scheme fixed the test but cements the class-A treatment. +- **Recommendation**: leave class A for now. Revisit when a real + host-override use case appears. + +### Q2. `water_species` — class A or class B? + +- The current `is_match` check on `water_species` treats it as + identity-defining (class A semantics). But the actual *meaning* of + the bit is mostly host bookkeeping ("does the dycore treat this as + water?"). CAM-SIMA has a `set_water_species` wrapper and uses it. +- **Recommendation**: class B, with the caveat that schemes whose + numerics depend on a constituent *being* water should declare that + in metadata as a hard requirement (different mechanism — not the + `is_match` machinery). + +### Q3. `mixing_ratio_type` — class A or class B? + +- The scheme's calculations assume `wrt_dry` or `wrt_moist`; this + feels class A. +- But hosts using different dycores might want to interpret the + same `std_name` differently — feels class B. +- **Recommendation**: class A. The mismatch should manifest as + different `std_name`s (`cloud_liquid_dry_mixing_ratio` vs + `cloud_liquid_wet_mixing_ratio`), not the same name with a runtime + override. Need cam-sima input. + +### Q4. After `is_match` relaxation: what happens on disagreement? + +- If two registrations of the same std_name agree on class A but + disagree on class B (e.g., `advected=.false.` from a scheme, + `advected=.true.` from the host), the second registration's class + B values should win without error. Effectively: the host overrides + the scheme. +- Order matters: today the host appends *after* the dynamic + constituents. Should we reverse so the host appends *first*? + Probably not — the "first registration wins on class A; host + setters override class B" model is conceptually clearer. +- **Recommendation**: silently dedup on matching class A; for class + B disagreements, the *later* registration's class B values are + ignored. Hosts use setters to override after registration + finalizes. + +### Q5. Should `%instantiate` accept class-B args at all? + +- **Option Y**: keep `%instantiate` accepting class B args (with + defaults). Schemes can supply them as hints; hosts can override. + Backward-compatible. +- **Option N**: remove class-B args from `%instantiate`. Schemes + *must* leave them to the host. Breaks the 4 cam-sima + scheme-registering schemes. +- **Recommendation**: option Y. The cost of breaking 4 schemes for + marginal clarity isn't worth it. + +### Q6. `ccpp_scheme_utils` singleton + +- Today: `ccpp_initialize_constituent_ptr(const_obj)` stores one + pointer module-wide. First instance wins. +- Fix options: + - (a) Maintain a per-instance pointer table; threading + `instance_number` through `ccpp_constituent_index`. + - (b) Document the limitation, route around it (no scheme uses + `ccpp_constituent_index` under multi-instance — capgen + already enforces `index_of_` everywhere). +- **Recommendation**: (b). It's a one-line doc note and zero code + change. + +### Q7. The `_layer` suffix — was a parallel `_interfaces` storage ever intended? (raised 2026-06-25, walkthrough prep) + +- **Observation.** The per-instance constituent object stores values and + tendencies as `%vars_layer(:,:,:)` and `%vars_layer_tend(:,:,:)` + (`src/ccpp_constituent_prop_mod.F90:167-168`), both allocated over the full + constituent axis (`:1558`, `num_values()` = every registered constituent). + The `_layer` qualifier in the names implies an anticipated **parallel + interface storage** (`vars_interface` / `vars_interface_tend`) that was + never added. +- **Evidence it was anticipated, not accidental.** The type carries + `is_layer_var` (`:637`, tests `vertical_layer_dimension`) **and** + `is_interface_var` (`:652`, tests `vertical_interface_dimension`) + predicates — so the design already distinguishes layer- vs + interface-located constituents, but only layer storage exists. +- **Latent gap.** A constituent declared on `vertical_interface_dimension` + has no storage slot today; `is_interface_var` would return true but there + is nowhere to put it. Whether any host/scheme actually needs interface + constituents is unknown (CAM-SIMA audit in §3 did not surface one). +- **Questions for discussion.** (a) Was `_interfaces` intended and dropped, or + is `_layer` just a (now-misleading) name? (b) Does any consumer need + interface-level constituents? (c) If **no** → drop the `_layer` suffix to + simplify; if **yes** → add the parallel `vars_interface` / `_tend` arrays and + route `is_interface_var` constituents to them. + +### Q8. Should a constituent always be a triplet — base + tendency + index? (raised 2026-06-25, walkthrough prep) + +- **Today (per `constituents.md` four rules):** a constituent is **not** a + forced triplet. It is *one registered base* (Rule 1, the only declaration + path), *zero-or-more optional* `tendency_of_` references (Rule 3 — a + scheme emits one only if it has a tendency), and a *framework-derived* + `index_of_` (never user-declared; filled at init via `%const_index`). +- **But storage already half-implies the triplet.** `%vars_layer_tend` is + allocated over the **whole** constituent axis (`:1558`), so every + constituent has a tendency *column* whether or not any scheme writes it. + So the "triplet" is already true at the **storage** level, but optional at + the **metadata/registration** level. +- **Question for discussion.** Should registration *force* the triplet + (declare base + tendency + index together, uniformly)? + - *For:* uniform mental model; matches the storage; removes the "did anyone + register a tendency?" ambiguity; could let the resolver validate + tendency producers against registered bases. + - *Against:* many constituents have no physics tendency (the column is + already there regardless, so forcing a declaration buys little); the + index is implicit by design and exposing it as a required member + re-introduces the index bookkeeping capgen deliberately hid; the + base is the only thing that *must* be registered. + - *Open sub-question:* if not forced, should the resolver at least **warn** + when a `tendency_of_` is produced for an `` that no register scheme + declared? (relates to §4.9 — no codegen-time cross-check of registration.) + +--- + +## 8. Three proposals — minimal / clean / deep + +### Proposal A — bugfix only + +**Scope**: +- Land the `ccpt_deallocate` ownership fix (done 2026-05-12). +- Update `scripts/constituents.py` for original capgen's auto-clone + path to pass `owned=.true.` (done). +- Add the three missing setters (`set_advected`, + `set_diagnostic_name`, `set_default_value`) without changing + semantics. Doesn't touch `is_match` or `%instantiate`. +- Document the gaps in `doc/constituents.md`. + +**Cost**: ~50 lines framework code + tests. No cam-sima changes +required. + +**Benefit**: closes the immediate bug, gives hosts the override +mechanism they need today (specifically for `diag_name`), unblocks +the advection test's deferred-property pattern. + +**Limit**: leaves `is_match` strict — hosts that disagree with a +scheme on `advected` still hit the "incompatible constituent" error. + +### Proposal B — class A/B split + setters + +**Scope** (in addition to A): +- Relax `is_match` to check only class A (`units` and possibly + `mixing_ratio_type`). +- Make all class-B properties optional in `%instantiate` with sane + defaults; deprecate (but keep accepting) class-B kwargs. +- Adopt the recommendation in Q4: silently dedup; host setters + override. +- Update `doc/constituents.md` with the register-then-override + workflow. +- (capgen) Reject `diagnostic_name` on `is_constituent=True` + scheme args at parse time, or downgrade it to a default-only hint. +- (capgen) **DONE 2026-05-13**: replaced `_FRAMEWORK_CONST_DIM_INPUTS` + with a `ResolvedArg.used_const_dim_std_names` field. + +**Cost**: ~150 lines framework + ~50 lines capgen + tests. +CAM-SIMA host code can stay as-is (the 4 scheme-side registrations +continue to work with their existing class-B values; they're just +not enforced anymore). Optional: tidy the 4 schemes to pass class-A +only. + +**Benefit**: physics schemes become genuinely portable across +hosts. The class-B override pattern that CAM-SIMA already uses for +`thermo_active` and `water_species` generalizes. + +**Limit**: does not change the registration model (still +explicit-only in capgen, still auto-clone in original capgen). + +### Proposal C — host-only registration + +**Scope** (in addition to B): +- Move the 4 cam-sima scheme-side register calls into a CAM-SIMA + helper module called from `cam_comp.F90`'s initialization. +- Drop register-phase `ccpp_constituent_properties_t(:)` support + from capgen (and possibly original capgen). Schemes only + consume constituents; only the host registers. +- Codegen-time enforcement: any `advected=true` scheme arg whose + std_name is not in the host's enumeration → codegen error. +- Eliminates the `_dynamic_constituents` per-suite buffer + entirely. + +**Cost**: ~300 lines code total; requires coordinated PRs across +ccpp-framework, ccpp-capgen, ccpp-capgen, atmospheric_physics, and +CAM-SIMA. The 4 schemes need their `_register` routines deleted (or +made no-ops); the host needs a new helper. + +**Benefit**: one source of truth for what constituents exist +(the host). Removes the auto-clone / scheme-register conceptual +overlap. Simplifies generator and runtime. + +**Limit**: changes the contract for the 4 scheme authors. Risk of +breaking yet-undiscovered downstream users of the scheme-side +registration model. + +### Comparison + +| Aspect | A | B | C | +|---|---|---|---| +| Lines changed | ~50 | ~200 | ~500+ | +| Coordination needed | framework only | framework + capgen | framework + both generators + cam-sima | +| Fixes the crash | yes | yes | yes | +| Fixes `diag_name` portability | yes (host overrides) | yes | yes | +| Relaxes `is_match` | no | yes | yes | +| Removes scheme-side register | no | no | yes | +| Risk to existing CAM-SIMA workflows | none | low | medium | + +### Recommendation + +**Adopt A immediately (mostly done), aim for B over the next 4–6 +weeks, table C until the framework PR for B is in and we have a +clearer signal on whether the scheme-side register pattern is worth +keeping.** + +--- + +## 9. Appendix: framework setter inventory + +(For reference during the meeting. Reproduced from +`design_constituents_mutability.md`.) + +`ccpp_constituent_properties_t` methods (`src/ccpp_constituent_prop_mod.F90`): + +``` +Instantiation + procedure :: instantiate ! takes std_name, long_name, diag_name (REQUIRED), + ! units, vertical_dim, plus optional + ! advected, default_value, min_value, molar_mass, + ! water_species, mixing_ratio_type + procedure :: deallocate + +Getters (subset) + procedure :: standard_name + procedure :: long_name + procedure :: diagnostic_name + procedure :: units + procedure :: vertical_dimension + procedure :: is_advected + procedure :: is_thermo_active + procedure :: is_water_species + procedure :: is_mass_mixing_ratio + procedure :: is_volume_mixing_ratio + procedure :: is_number_concentration + procedure :: is_dry / is_moist / is_wet + procedure :: minimum + procedure :: molar_mass + procedure :: default_value + procedure :: has_default + procedure :: is_framework_owned ! NEW 2026-05-12 + +Setters (changes after instantiate) + procedure :: set_const_index + procedure :: set_thermo_active + procedure :: set_water_species + procedure :: set_minimum + procedure :: set_molar_mass + procedure :: set_framework_owned ! NEW 2026-05-12 + procedure :: set_advected ! GAP + procedure :: set_diagnostic_name ! GAP + procedure :: set_default_value ! GAP (or keep class A) + procedure :: set_mixing_ratio_type ! GAP (if class B) + +Identity / equality + procedure :: equivalent ! full equality + procedure :: is_match ! checks units + (class-B props ← too strict) +``` + +`ccpp_constituent_prop_ptr_t` is the pointer wrapper. Has parallel +setters that delegate to the underlying `ccpp_constituent_properties_t`. + +--- + +## Cross-references + +- `doc/constituents.md` — capgen's user-facing constituents reference. +- `design_constituent_api.md` (memory) — capgen's per-instance option-A design. +- `design_constituents_mutability.md` (memory) — extended design notes incl. class A/B classification. +- `project_implementation_status.md` (memory) — current implementation state and deferred items. +- `scripts/constituents.py` — original capgen's host-cap generator. +- `src/ccpp_constituent_prop_mod.F90` — framework. +- `capgen/generator/host_constituents.py` — capgen's host-side module emitter. +- `capgen/generator/suite_resolver.py` (`_resolve_constituent_arg`) — capgen's resolver routing. +- `EXT/cam-sima/CAM-SIMA/src/physics/utils/cam_constituents.F90` — CAM-SIMA's host-side wrappers around framework setters. + diff --git a/doc/file_catalogue_DRAFT.md b/doc/file_catalogue_DRAFT.md new file mode 100644 index 00000000..778ea16a --- /dev/null +++ b/doc/file_catalogue_DRAFT.md @@ -0,0 +1,149 @@ +# capgen repository — file catalogue (DRAFT) + +> **Status: temporary draft for the code-walkthrough prep.** One row per file, except +> the many test/example *input* fixtures, which are collapsed. Once reviewed, the +> relevant sections will be folded into `README.md` / `doc/DevelopersGuide/`. +> External checkouts under `EXT/` (UFS reference + capgen integration trees) are +> intentionally excluded — they are not part of this repository. + +## Top level + +| File | Description | +|------|-------------| +| `README.md` | Repository overview and entry point. | +| `LICENSE` | License. | +| `end-to-end-tests.sh` | Driver script that builds and runs all end-to-end tests. | +| `ccpp_constituent_prop_mod.F90.patch` | Patch applied to the runtime constituent-properties module for host integrations. | +| `CODEOWNERS`, `.codecov.yml`, `.codee-format`, `.gitignore` | Repo/CI configuration (code owners, coverage, formatter, ignore rules). | +| `.github/` | GitHub Actions CI workflows (unit tests, end-to-end tests, doxygen). | + +## `capgen/` — command-line entry points + +| File | Description | +|------|-------------| +| `__init__.py` | Package marker (“next-generation CCPP code generator”). | +| `ccpp_capgen.py` | **Main generator CLI.** Parses metadata + the SDF, resolves variables, and writes the caps, `ccpp_kinds.F90`, and `datatable.xml`. Hosts flags like `--kind-type`, `--trace`, `--no-host-introspection`, and the compat shims. | +| `ccpp_datafile.py` | CLI to query the generated `datatable.xml` (generated files, scheme files, dependencies) for build systems / CMake. | +| `ccpp_validator.py` | **Standalone validator** — checks scheme Fortran source against its `.meta` (intent/type/kind/rank/dimensions). Separate tool from the generator; owns the one Fortran parser. | + +## `capgen/generator/` — cap code generation + +| File | Description | +|------|-------------| +| `__init__.py` | Package marker. | +| `datatable.py` | Writes/reads `datatable.xml` mapping suites → generated files, scheme modules, and dependencies (the build-system interface). | +| `suite_xml.py` | Parses the Suite Definition File (SDF) XML into the suite object model (groups, subcycles, subcolumns). | +| `suite_types.py` | Object model for suites/groups/schemes, incl. intrinsic-vs-external scheme classification. | +| `suite_resolver.py` | Resolves a suite end-to-end: matches variables across schemes + host, constituents, index symbols, unit normalization. | +| `suite_cap.py` | Emits the **suite-level cap** (`ccpp_physics_run`/`_init`/… dispatching to groups; register-before-init contract). | +| `group_cap.py` | Emits the **per-group caps** that call the schemes, with argument marshalling and inline transforms. | +| `host_cap.py` | Emits the **host cap** (registration + runtime introspection API; introspection routines stubbed under `--no-host-introspection`). | +| `host_constituents.py` | Host-side constituent handling (`type=host` constituent tables). | +| `suite_data.py` | Emits the generated suite **data module** — pointer-wrapper DDTs plus transform local temporaries. | +| `kinds_writer.py` | Writes `ccpp_kinds.F90` (kind definitions the caps `use`). | +| `trace.py` | Shared helpers emitting the gated `if (trace) write(...) 'CCPP TRACE …'` lines in every cap (toggled by `--trace`). | + +## `capgen/metadata/` — metadata parsing & variable resolution + +| File | Description | +|------|-------------| +| `__init__.py` | Package marker. | +| `metadata_table.py` | Parser for `.meta` metadata-table files (`[ccpp-table-properties]` / `[ccpp-arg-table]`). | +| `variable_resolver.py` | Core variable matching/transform engine — unit + kind conversions, vertical flip, DDT typing. | +| `unit_conversion.py` | Unit-conversion formula table (`{var}` substitution) feeding the auto-inserted unit transforms. | +| `registered_dimensions.py` | Registry of count-dim ↔ index-var pairings (`SCALAR_INDEX_DIMS`) and framework count dimensions. | +| `legacy_compat.py` | **Transient shim** — rewrites legacy CCPP standard names (e.g. `horizontal_loop_extent`) at parse time. | +| `dim_aliases.py` | **Transient shim** — collapses equivalent GFS-physics dimension names. | +| `auto_clone_constituents.py` | **Transient shim** — reinstates original-capgen auto-cloning of static constituents. | + +## `capgen/metadata/parse_tools/` — shared parse utilities + +| File | Description | +|------|-------------| +| `__init__.py` | Package marker. | +| `parse_source.py` | Parsing primitives: parse context + exception types. | +| `parse_checkers.py` | Metadata field validators (`check_units`, `check_dimensions`, `check_cf_standard_name`, …). | +| `parse_log.py` | Shared logging utilities for parse processes. | +| `io_helpers.py` | File-write helpers with write-if-changed (no-op-if-unchanged) semantics. | +| `fortran_conditional.py` | Builds Fortran conditional expressions (in local names) for active/optional-argument handling. | +| `xml_tools.py` | XML helpers — entity expansion and pretty-printed writing (SDF / datatable). | + +## `capgen/schema/` & `capgen/src/` — schema + shipped runtime Fortran + +| File | Description | +|------|-------------| +| `schema/suite_v1_0.xsd` | XML schema for SDF v1.0. | +| `schema/suite_v2_0.xsd` | XML schema for SDF v2.0 (adds suite-level ``/``). | +| `src/ccpp_constituent_prop_mod.F90` (+ `.meta`) | Runtime constituent-properties DDT module shipped with the framework. | +| `src/ccpp_hash_table.F90` | Runtime hash-table support. | +| `src/ccpp_hashable.F90` | Hashable base type used by the hash table. | +| `src/ccpp_scheme_utils.F90` | Runtime scheme utility routines. | + +## `unit-tests/` — pytest suite (one row per driver; fixtures collapsed) + +| File | Description | +|------|-------------| +| `run_tests.py`, `conftest.py`, `__init__.py` | Test runner, pytest fixtures, package marker. | +| `test_metadata_table.py` | Tests for `metadata/metadata_table.py`. | +| `test_variable_resolver.py` | Tests for `metadata/variable_resolver.py`. | +| `test_registered_dimensions.py` | Tests for `metadata/registered_dimensions.py`. | +| `test_dim_aliases.py` | Tests for `metadata/dim_aliases.py`. | +| `test_legacy_compat.py` | Tests for `metadata/legacy_compat.py`. | +| `test_auto_clone_constituents.py` | Tests for `metadata/auto_clone_constituents.py`. | +| `test_io_helpers.py` | Tests for `parse_tools/io_helpers.py`. | +| `test_suite_xml.py` | Tests for `generator/suite_xml.py`. | +| `test_suite_types.py` | Tests for `generator/suite_types.py`. | +| `test_suite_resolver.py` | Tests for `generator/suite_resolver.py`. | +| `test_suite_cap.py` | Tests for `generator/suite_cap.py`. | +| `test_suite_data.py` | Tests for `generator/suite_data.py`. | +| `test_host_cap.py` | Tests for `generator/host_cap.py`. | +| `test_host_constituents.py` | Tests for `generator/host_constituents.py`. | +| `test_kinds_writer.py` | Tests for `generator/kinds_writer.py`. | +| `test_datatable.py` | Tests for `generator/datatable.py`. | +| `test_trace.py` | Tests for `generator/trace.py`. | +| `test_ccpp_datafile.py` | Tests for `ccpp_datafile.py`. | +| `test_validator.py` | Tests for `ccpp_validator.py` (incl. the Fortran parser). | +| `test_control_validation.py` | Tests for control-variable validation rules. | +| `test_integration.py` | End-to-end generator integration tests (full parse → resolve → emit). | +| `sample_files/`, `sample_suite_files/` | **~100 metadata / SDF / Fortran fixtures** consumed by the tests above — not catalogued individually. | + +## `end-to-end-tests/` — full build-and-run cases (one row per case; fixtures collapsed) + +Each case directory bundles host + scheme Fortran, `.meta`, an SDF, a `*_test_reports.py` +comparison driver, and CMake glue. The fixtures are collapsed; the row describes what the case exercises. + +| Case | What it exercises | +|------|-------------------| +| `capgen/` | **Overall generator capabilities** — multiple suites & groups, DDT usage (incl. an undocumented DDT member), `ccpp_constant_one:N` and bare-`N` dimensions, non-standard/integer dimensions, variables promoted to suite level, dimensions set in the register phase and used to allocate module-level interstitials, and threading. | +| `advection/` | Constituent advection — cloud liquid/ice constituents with tendency application (`apply_constituent_tendencies` invoked twice); includes a deliberate error suite (`cld_suite_error.xml`) to exercise diagnostics. | +| `advection_auto_clone/` | Same fixtures as `advection/`, run through the `--legacy-auto-clone-constituents` shim path. | +| `ddthost/` | A host whose CCPP data is carried in a derived type (`host_ccpp_ddt`); runs the temp + DDT suites against it. | +| `var_compat/` | The variable-compatibility object (`VarCompatObj`): unit conversions (forward & reverse), vertical flip (`top_at_one`), kind conversions, and combinations — plus subcycles (nested, dynamic vs fixed iteration length, shared length-defining standard names). | +| `nested_suite/` | Nested suites (a suite that includes a sub-suite), expanded at and inside groups; SDF schema **2.0**; suite-level single ``/`` schemes. Inherited from `var_compat`. | +| `constituents_dim/` | Variables dimensioned by the framework constituent count `number_of_ccpp_constituents` (host never declares it); covers host-owned, framework-allocated, and scheme-allocated count-dim cases, plus consuming constituents without re-flagging (rule b). | +| `suite_allocate/` | A suite-owned, **scheme-allocated** (`allocatable`) variable promoted to `ccpp__data`; its dimension is also suite-owned and set in the `timestep_init` phase (so it can't be allocated at init). | +| `instances/` | **Multiple model instances** — `instance`/`number_of_instances` paired control; the host loops `ccpp_physics_run` over instances with per-instance data (unit-conversion schemes are just the vehicle). | +| `instances_advection/` | Multiple instances combined with constituent advection — scheme-registered constituents with a per-instance buffer (`ninstances` × cloud-liquid + tendency application). | +| `opt_arg/` | Optional-argument handling — present/absent dummy arguments (pointer association vs runtime guard). | +| `chunked_data/` | Chunked/blocked host data — `chunk_begin`/`chunk_end` bounds over `nchunks` chunks. | +| `*_test_reports.py` (where present) | Per-case driver that builds, runs, and diffs expected vs actual output (older cases; newer cases run via `ctest`/CMake). | +| `CMakeLists.txt`, `cmake/`, `utils/` | Shared CMake configuration and helpers for the e2e harness. | + +## `doc/` — documentation + +| File | Description | +|------|-------------| +| `README.md` | Documentation index. | +| `redesign_prompt.md`, `redesign_analysis.md`, `redesign_analysis_original_*.md` | Original redesign brief and analysis that motivated capgen. | +| `briefing.md`, `briefing_pm.md` | Design briefings. | +| `migration.md` | Guide for migrating a host from ccpp-prebuild/original-capgen to capgen. | +| `capgen_compat_layer.md` | Documents the transient compatibility shims (legacy names, dim aliases, auto-clone). | +| `constituents.md` | Constituent-handling design. | +| `constituents_overhaul.md` | Proposed constituent-model overhaul (proposals A/B/C). | +| `auto_clone_constituents.md` | Design notes for the auto-clone-constituents shim. | +| `cam4_fwaut_constituent_order.md` | Case study: CAM4 FWAUT constituent-ordering b4b investigation. | +| `Doxyfile.in` | Doxygen configuration. | +| `CMakeLists.txt` | Build glue for the docs. | +| `DevelopersGuide/` | Developers Guide (`README.md`, generated PDFs, LaTeX style) — bundle, not catalogued per file. | +| `HelloWorld/` | Worked “hello world” host + scheme + suite + build example — bundle, not catalogued per file. | +| `img/` | Documentation images. | diff --git a/doc/migration.md b/doc/migration.md new file mode 100644 index 00000000..7673b472 --- /dev/null +++ b/doc/migration.md @@ -0,0 +1,1297 @@ +# Migrating from ccpp-prebuild / ccpp-capgen to capgen + +This document captures the **user-facing differences** a host model author +or scheme author needs to know when moving metadata, suite XML, and host +Fortran from the legacy ccpp-prebuild + ccpp-capgen toolchain to +**capgen**. It complements `doc/redesign_prompt.md` (design spec) and +`doc/redesign_analysis.md` (analysis of the old systems). + +*Last revised: 2026-06-05.* Current unit-test suite: 1516 passing. + +**Repository layout** (post-2026-05-13 cleanup): tooling lives under +`capgen/` (top-level of this repo). Unit tests live at the top +level in `unit-tests/`; end-to-end tests in `end-to-end-tests/`. Run +the unit suite from the repo root with `python -m pytest unit-tests/`. + +## Table of contents + +1. [Metadata format changes](#1-metadata-format-changes) + 1. [1.8 `horizontal_loop_extent` → `horizontal_dimension`](#18-deprecated-standard-names-rewritten-by---legacy-mode) + 2. [1.10 GFS-physics vertical-dim aliases (`--gfs-dim-aliases`)](#110-gfs-physics-vertical-dim-aliases---gfs-dim-aliases) +2. [Suite definition file (SDF) changes](#2-suite-definition-file-sdf-changes) +3. [Host Fortran requirements](#3-host-fortran-requirements) +4. [Generator CLI and build integration](#4-generator-cli-and-build-integration) +5. [Generated cap layout — what's new and what changed](#5-generated-cap-layout--whats-new-and-what-changed) +6. [Framework changes (constituents)](#6-framework-changes-constituents) + 1. [6.3 Host metadata wins over auto-provisioning](#63-host-metadata-wins-over-auto-provisioning-2026-05-12) + 2. [6.4 Legacy auto-clone registration (`--legacy-auto-clone-constituents`)](#64-legacy-auto-clone-registration---legacy-auto-clone-constituents) +7. [Validator (`ccpp_validator.py`)](#7-validator) +8. [Known gaps and deferred items](#8-known-gaps-and-deferred-items) + +--- + +## 1. Metadata format changes + +### 1.1 Table types + +Four `type =` values in `[ccpp-table-properties]`: + +| Type | Contents | +|---------|---------------------------------------------------------| +| `control` | Control variables passed as ``ccpp_physics_*`` args. | +| `host` | Host-model variables imported via `use`. | +| `ddt` | Derived-type definitions. | +| `scheme` | Scheme metadata. | + +The legacy `type = module` (capgen) becomes `type = host`. The legacy +`TYPEDEFS_NEW_METADATA` Python dict (prebuild) is replaced by `type = ddt` +tables. See `doc/redesign_prompt.md` §3.2. + +### 1.2 New table-property attributes + +All optional inside the `[ccpp-table-properties]` block: + +| Attribute | Applies to | Description | +|-----------------------|-----------------------|-------------| +| `module_name` | scheme, host, ddt | Fortran module name; overrides "module name = table name" when they differ. | +| `dependencies` | any | Comma-separated list of file paths to compile. **May appear multiple times** in one block (new this session); single occurrence still accepted. | +| `dependencies_path` | any | Relative base for `dependencies` entries. Single-valued. | +| `source_path` | any | Relative path to the Fortran source. Single-valued. | +| `kind_spec` | any | `:=>spec` (or shorthand). May appear multiple times. | + +Example with multi-line dependencies (real CCPP physics pattern): + +``` +[ccpp-table-properties] + name = GFS_rrtmg_setup + type = scheme + module_name = GFS_rrtmg_setup # optional when names match + dependencies_path = ../../ + dependencies = tools/mpiutil.F90 + dependencies = hooks/machine.F + dependencies = Radiation/RRTMG/radlw_main.F90,Radiation/RRTMG/radsw_main.F90 +``` + +> **Standalone DDT files require `module_name`.** A `type = ddt` table +> in a `.meta` file with **no co-located** `scheme`/`host`/`control` +> table (a "wrapper object" like `ccpp_optical_props.meta` defining +> `ty_optical_props_1scl_ccpp`) cannot inherit its module from a sibling. +> If the defining Fortran module name differs from the DDT table (type) +> name — which it almost always does for these wrappers — you **must** +> declare `module_name` explicitly. capgen does *not* guess (e.g. +> from the file name); a DDT it can't resolve raises a clear error at +> generation time naming the type and the `module_name` remedy. + +### 1.3 New per-variable attributes + +Inside a `[ var_name ]` section. All optional. + +| Attribute | Type | Default | Notes | +|------------------|------|---------|-------| +| `top_at_one` | bool | `False` | When host and scheme disagree, generator emits a vertical-flip transform with reverse-stride subscript on the host side. Meaningless on variables without a vertical dimension. | +| `constituent` | bool | `False` | Scheme metadata only. Marks the var as a constituent reference. | +| `advected` | bool | `False` | Scheme metadata only. | +| `molar_mass` | float | `0.0` | Scheme metadata only. | +| `diagnostic_name` | str | (defaults to `local_name`) | Host-tooling hint; mutually exclusive with `diagnostic_name_fixed`. | +| `allocatable` | bool | `False` | Must match the Fortran `allocatable` attribute on the dummy. Required for any array the *scheme* allocates (see §1.3.3). | + +#### 1.3.1 Host `active` + scheme arg shape + +When a host variable carries `active = ()`, the host's +contract with the cap is "this variable's storage is only valid when +the condition holds". capgen honors that contract differently +depending on the matching scheme arg's optionality: + +**Scheme arg is `optional = True`** — the cap uses pointer association +so the scheme observes `PRESENT()` according to the active condition: + +```fortran +if () then + ptr%ptr => () +else + nullify(ptr%ptr) +end if +call scheme(..., my_arg=ptr%ptr, ...) +``` + +**Scheme arg is non-optional** — the scheme is asserting the variable +is mandatory. The cap emits a runtime guard before the call so an +inactive-but-required variable surfaces as a clean error rather than a +silent read of unallocated memory: + +```fortran +if (.not. ()) then + errmsg = "scheme 'X' phase 'Y' requires variable '' but " & + // "host active condition () is false" + errflg = 1 + return +end if +call scheme(..., my_arg=(), ...) +``` + +The guard runs before any unit/kind/vertical-flip transform pre-call +code, so transforms never see invalid host memory. Multiple required +arguments with `active` conditions each get their own guard block — one +per arg keeps the error messages targeted. + +It is the suite designer's responsibility to schedule the call so the +host's active condition holds when a required-arg scheme runs. The +guard converts violations from latent runtime bugs into immediate +errflg/errmsg returns. + +> **Earlier (relaxed 2026-05-20)**: a previous iteration of this rule +> rejected `active` + non-optional pairings at resolution time and +> required scheme metadata to declare `optional = True`. That forced +> scheme metadata to misrepresent the Fortran for schemes that +> legitimately require a host-conditional variable. The current rule +> defers the check to runtime and leaves the metadata honest. + +#### 1.3.2 Host/scheme metadata cross-checks + +The resolver enforces three cross-metadata checks per scheme arg +against its defining source (host metadata or, for suite-owned +variables, the first scheme to write the var with `intent=out`): + +| Aspect | Rule | Notes | +|---|---|---| +| **Type identity** | Strict string match after `strip().lower()`. | No coercion across `real` / `integer` / `logical` / DDT. DDT names match identically; `external:m:t` matches `external:m:t`. | +| **Rank** | `len(host.dimensions) == len(scheme.dimensions)`. | A scheme that asks for `(horizontal_dimension)` while the host declares `()` is rejected. | +| **Per-position dimension identity** | Each entry is canonicalized to `lower:upper`; strict match per position. | See "default lower bound" below. | + +**Default lower bound — three equivalent spellings:** + +- bare `foo` (no explicit lower bound) +- `1:foo` (integer literal one) +- `ccpp_constant_one:foo` (the standard name) + +All three collapse to a single canonical representative, so the host +declaring `(vertical_layer_dimension)` matches the scheme declaring +`(1:vertical_layer_dimension)` and vice versa. + +**Every other lower bound is distinct.** `2:nlev` is not the same +axis as `1:nlev`; `start_idx:nlev` is not the same as +`ccpp_constant_one:nlev`. Different lower bound describes a +different sub-range and must be spelled identically on both sides. + +**No upper-bound name aliasing.** `horizontal_dimension` and +`horizontal_loop_extent` are different names at the resolver layer. +The `--legacy-mode` shim (see §3) rewrites legacy names at parse +time when enabled; without that shim the legacy spellings should not +appear in metadata at all. + +**Numeric kind is *not* checked here.** Host `kind_phys` vs scheme +`real32` silently triggers the transform-copy pipeline (§5.3). This +is deliberate: real CCPP-physics schemes legitimately mix precisions +and rely on the cap to handle the copy. Watch for unintended +narrowing — there is no static guard. **Character `len=`** has its +own block: matching `len=N` values pass, mismatched specific lengths +are an error unless the *consuming* scheme uses `len=*` (wildcard). +`len=*` is only valid where a variable is *passed*, never where its +storage is *defined*: host and DDT metadata must give every character +variable a concrete length, and so must the first `intent=out` scheme +that defines a suite-owned character variable (see below). Both are +rejected with a clear error rather than emitting an undeclarable +`character(len=*)` component. Control variables are exempt — they are +pass-through dummy arguments (`suite_name`, `errmsg`, …) the caps +legitimately declare `character(len=*)`. + +**Suite-owned variables.** The first scheme to write a standard +name with `intent=out` (in phase→scheme order) freezes the var's +type/kind/dimensions/units on the SuiteVar; every later scheme that +consumes it goes through the same checks against the frozen fields. +Because that first writer *defines* the storage the framework +allocates in `ccpp__data`, a character definer must declare a +concrete length (`kind = len=N`); `len=*` there is an error. Later +consumers/writers of the same variable may use `len=*` as a wildcard. +Error messages name the source as `host`, `control`, or `suite` so +you know whose contract you're violating. + +**Suite-owned storage is never default-initialized** — by design. +capgen emits the `ccpp__data` components with no default value. +An `intent(out)` argument is the scheme's contract to define that variable +on *every* return path; the framework will not paper over an unset output +the way original capgen's zero-initialized interstitials did. A ported +scheme that returns early (e.g. a `fixed_scon` branch) without assigning +one of its `intent(out)` dummies leaves the suite-owned storage undefined, +and a later consumer reads garbage (in a debug build, often a trap value). +This is a common porting hazard original capgen used to mask — audit +early-return paths for unset `intent(out)` args. + +#### 1.3.3 `allocatable` and who owns suite-data allocation + +A **suite-owned variable** (an interstitial: first written by a scheme +with `intent=out`, then consumed by another) is stored as a component +of the generated `ccpp__data` DDT. capgen allocates it for +you — **once**, in `suite_data_init_fields`, which runs at the very +start of `_init`. That works only when every dimension is known +that early, i.e. a **host variable** or a value set in the **`register`** +phase. + +When the size is *not* known at init — the array is dimensioned by a +quantity a scheme computes later (in `init` / `timestep_init` / `run`, +e.g. a per-timestep daylight-column count) — the suite cannot size it. +Such a variable must be declared **`allocatable`** and allocated by its +**producing scheme**: + +- metadata: `allocatable = True` on the arg; +- Fortran: `allocatable, intent(out)` on the dummy, and an explicit + `allocate(...)` in the scheme body (an `intent(out)` allocatable is + auto-deallocated on entry, so element assignment needs it allocated + first). + +For an `allocatable` arg capgen then: (1) does **not** pre-allocate it +in `init_fields`; (2) passes the **whole** component at call sites +(`...%var`, no array section — an allocatable/assumed-shape mismatch is +otherwise a compile error); and (3) still frees it in +`suite_data_final_fields` under an `if (allocated(...))` guard. So the +**scheme allocates, the suite (or the scheme) deallocates** — the guard +makes either order safe (no leak, no double-free). + +Size it with the **authoritative dimension variable** — the standard +name in the arg's `dimensions` — not a look-alike local or a derived +expression. If that dimension is a scheme-set quantity, pass it in as a +scalar `intent=in` arg and `allocate` with it. (Real example: an array +declared `number_of_vertical_interfaces_in_RRTMGP` must be sized with +that value, **not** the host's `vertical_interface_dimension` nor +`nlay+1`, which differ when the scheme runs on a reduced vertical grid.) + +**Generation-time guard.** capgen rejects a *non*-`allocatable` +suite-owned array whose dimension is written by a scheme in any phase +after `register` — it would otherwise be allocated from uninitialized +memory. The error names the variable, the offending dimension, and the +fix (declare it `allocatable`). The check is sound but partial: it sees +only dimensions a *scheme* writes, not a host scalar the host driver +re-computes each step — those remain the author's responsibility, and +the rule "anything the scheme allocates must be `allocatable` in Fortran +and metadata" is ultimately enforced by the compiler plus `ccpp_validator`. + +### 1.4 Sliced local names with long subscript indices + +Local names with array slices may carry CCPP standard names as subscript +tokens: + +``` +[ dqdt(:,:,index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array) ] + standard_name = ... +``` + +The 63-char Fortran-identifier limit is enforced only on the base +identifier (`dqdt`), not on subscript tokens (which are CCPP standard +names resolved at codegen time and routinely exceed 63 chars). + +### 1.5 Unit strings: bare vs explicit positive exponent + +`m2` and `m+2` (or any `` vs `+` +combo) are normalized internally and treated as equivalent. Pre-existing +unit-conversion entries don't need to be duplicated; either spelling +matches. + +### 1.6 Improved error messages + +- **Duplicate standard name**: error message now lists both colliding + access paths and hints at the "sibling DDT instance" pattern (when + applicable). +- **Subcycle bound unresolved**: error names the std_name and points + at the control/host metadata as the fix. +- **Instance-dim used without `instance_number`**: error explains the + paired-opt-in requirement (see §1.7). + +### 1.7 Paired-optional control pairs (instances and threads) + +There are two symmetric `(index, count)` control pairs: +`instance_number` / `number_of_instances` and `thread_number` / +`number_of_threads`. Both behave identically: + +- Declare **both** members in `type=control` → opt into that paired + (multi-instance / multi-threading) API. Both flow as control dummies + through every lifecycle and physics-phase signature. +- Declare **neither** → the single API. Public entry points drop both + args; where the index would appear the framework uses literal `1` + (and, for instances, internal per-instance arrays size to length 1). +- Declare exactly one of a pair → hard error from the validator. +- Declare either count in `type=host` → hard error (must be + `type=control`). +- Dimension a host variable by `number_of_instances` / + `number_of_threads` without declaring its pair → hard error (the + scalar-index collapse needs the index variable in scope; see §3.4). + +Hosts that need neither multi-instance nor multi-threading can drop +both pairs entirely. + +> Why symmetric: `instance_number` indexes framework-owned per-instance +> state, so `number_of_instances` is read at register/init to size it. +> `thread_number` indexes host-owned per-thread containers; the +> framework doesn't yet read `number_of_threads`, but it's carried as a +> control dummy so the framework can size per-thread state in future — +> exactly as it does for instances today. + +### 1.8 Deprecated standard names rewritten by `--legacy-mode` + +`--legacy-mode` is a transient migration shim that rewrites a small +set of deprecated standard names to their canonical capgen +equivalents at parse time. The full table currently covers: + +| Deprecated (legacy) | Canonical (capgen) | +|--------------------------------|--------------------------| +| `horizontal_loop_extent` | `horizontal_dimension` | +| `number_of_openmp_threads` | `number_of_threads` | + +Why each entry: + +* `horizontal_loop_extent` — ccpp-prebuild / original ccpp-capgen used + this for the horizontal-axis std name in scheme metadata. capgen + uses `horizontal_dimension` uniformly; the run-vs-non-run distinction + isn't expressed in scheme metadata anymore (host passes + `horizontal_loop_begin` / `horizontal_loop_end` as control vars and + the generated cap slices accordingly). +* `number_of_openmp_threads` — legacy CCPP-physics hosts (CCPP-SCM + 17p8 in particular) size per-thread DDT containers by + `number_of_openmp_threads` (e.g. `physics%Interstitial`). capgen + uses `number_of_threads`, which matches the `thread_number` control + variable, so the registered scalar-index dim table can substitute + `physics%Interstitial(thread_number)%…` automatically (see §3.4). + +The rewrite fires for both standard-name attributes AND dimension +tokens (so a host's `dimensions = (number_of_openmp_threads)` becomes +`dimensions = (number_of_threads)` before any further processing). + +Migration paths: + +1. **Edit the metadata** (recommended) — search-and-replace the + legacy names in every host / scheme `.meta` you maintain. +2. **Use `--legacy-mode`** (transient) — pass `--legacy-mode` to both + `ccpp_capgen.py` and `ccpp_validator.py` and the renames happen + at parse time. A loud warning banner prints at startup, listing + every pair the shim is rewriting, so the substitution is never + invisible. This shim *will be removed*; treat it as a runway, + not a destination. + +### 1.9 Inline comments + +`#` starts a comment **anywhere on a line**, not just at column 0. +Everything from the `#` to end-of-line is discarded before the +parser sees the rest of the line. Trailing whitespace left behind +by the strip is also removed, so section headers and key=value +lines parse cleanly. + +``` +[ ap_indices ] # legacy index slot + standard_name = ap_indices + units = count + dimensions = () # was (nap_indices) before the refactor + type = integer +``` + +No escape mechanism is provided — `#` is not a legitimate character +in any metadata value (units, kinds, identifiers, dim lists, +Fortran conditional expressions). `;` is still accepted as a +full-line comment marker (at column 0 after whitespace), matching +the historic blank-line convention, but is *not* treated as an +inline comment marker (`;` can plausibly appear inside a +`long_name`). + +### 1.10 GFS-physics vertical-dim aliases (`--gfs-dim-aliases`) + +GFS-physics scheme metadata uses two spellings for what is physically +the vertical-layer axis: + +- `adjusted_vertical_layer_dimension_for_radiation` (radiation schemes) +- `vertical_composition_dimension` (chemistry schemes) + +Both are the **same axis** as `vertical_layer_dimension` from the +host's point of view, but legacy hosts (CCPP-SCM 17p8, GFS) carry the +three names as **distinct host variables** (each addressable as its +own scalar dim std name) — so a parse-time substitution like +`--legacy-mode` would erase the variable behind the renamed token and +break host metadata. + +`--gfs-dim-aliases` collapses the three names **only inside the +resolver's per-position dimension-identity check** (upper bound only; +lower bounds never alias). The variables themselves stay distinct +everywhere else — `[ adjusted_vertical_layer_dimension_for_radiation ]` +remains its own host `type=control` entry; the access path in +generated code is unchanged; only the resolver's +"these dims describe the same axis" comparison treats the three names +as equivalent. + +Single touchpoint in the generator +(`generator/suite_resolver.py::_canonical_dim`); the validator does +not carry the flag (it never reaches the resolver's canonicaliser). +Self-contained module `metadata/dim_aliases.py`; every touchpoint +tagged `# dim-aliases:` for clean removal. + +Like `--legacy-mode`, this is a transient migration shim with a loud +startup banner — drop the GFS spellings from your scheme metadata in +favour of `vertical_layer_dimension` and the flag becomes unnecessary. + +--- + +## 2. Suite definition file (SDF) changes + +### 2.1 Schema v2.0 with nested-suite expansion + +Capgen parses v2.0 SDFs and expands `` references +recursively at parse time. See `doc/redesign_prompt.md` §3 and the +`suite_v2_0.xsd` schema. + +### 2.2 `` with CCPP standard-name loop bound + +```xml + + effr_pre + +``` + +The `loop=` attribute accepts: + +- **Integer literal** (`loop="3"`) — emitted verbatim. +- **CCPP standard name** (`loop="num_subcycles_for_effr"`) — resolved + against host/control metadata; supports DDT-component access paths + (e.g. `phys_state%num_subcycles`). +- **Absent / empty** — treated as `loop="1"`. + +The loop-bound standard name is automatically included in the +introspection inputs list (`ccpp_physics_suite_variables` and +`_suite_host_data`). + +### 2.3 Nested `` elements + +```xml + + + effr_calc + + +``` + +Nested subcycles produce nested `do` loops in the generated cap. Loop +counter variables follow the convention: + +- Outermost / single-level: `ccpp_loop_counter`. +- Each deeper level: `ccpp_loop_counter_2`, `ccpp_loop_counter_3`, ... + +Effective iteration count = product of every level's `loop=` value. +`effr_calc` in the example runs 3·2 = 6 times. + +### 2.3.1 Passing the loop counter / extent to a scheme + +A scheme inside a `` block may consume the current iteration +counter and the total iteration count via two CCPP standard names: + +| Standard name | Fortran type | Meaning | +|----------------------|--------------|----------------------------------------------------------| +| `ccpp_loop_counter` | integer | Current subcycle iteration (1 … `ccpp_loop_extent`) | +| `ccpp_loop_extent` | integer | Total iterations — the `loop=` value on the `` | + +These are **loop-context control variables**: the host model does **not** +declare them. capgen emits them automatically as locals in the +generated group cap (the `do` loop's induction variable for the counter, +the loop bound for the extent), and resolves any scheme arg requesting +them against those locals. + +Example scheme metadata fragment: + +``` +[iter] + standard_name = ccpp_loop_counter + units = index + dimensions = () + type = integer + intent = in +[niter] + standard_name = ccpp_loop_extent + units = index + dimensions = () + type = integer + intent = in +``` + +Place the scheme in a `` in the SDF: + +```xml + + sfc_diff + GFS_surface_loop_control_part1 + sfc_nst + +``` + +The generated group cap will emit `do ccpp_loop_counter = 1, 2` and call +the scheme with `iter = ccpp_loop_counter, niter = 2` (or the loop's +resolved local name when `loop=` is used). + +**Scope is the subcycle body.** A scheme that requests +`ccpp_loop_counter` / `ccpp_loop_extent` but is NOT inside a +`` block raises a clear parse-time error pointing at this +contract. + +**Nested-subcycle nuance** (see §8): nested-subcycle schemes that ask +for `ccpp_loop_counter` currently get the **outermost** loop's counter, +not the innermost. None of the in-tree physics catalogs use the +inner-counter case yet; revisit when one needs it. + +### 2.4 Suite-level `` and `` schemes + +```xml + + my_init_scheme + ... + my_final_scheme + +``` + +- Each element contains a **single** scheme name as text content. + Multiple `` children inside ``/`` is a schema + violation. (Group-shaped lists belong inside ``.) +- The named scheme's `init` / `final` phase metadata is resolved like + any other scheme phase; missing-phase metadata is a generator error. +- The scheme call is emitted inside `_init` / `_final` + with USE for the scheme module + per-arg host modules, and the + standard errflg check. +- Call ordering: + - `_init`: after all group `state_alloc` and + `suite_data_init_fields`, **before** the `CCPP_SUITE_FRAMEWORK_INITIALIZED` + state transition. + - `_final`: before the `CCPP_SUITE_UNREGISTERED` transition. + +**Accepted spellings**: `` and `` only. Legacy spellings +**``** (typo), **``** (correct long form), and +**``** are rejected with a clear error pointing at the +canonical short form. + +To exercise: + +1. Declare a scheme with `init` and/or `final` phases in its metadata + (minimal sig — just `errmsg` + `errflg` — is fine). +2. Reference it in the SDF as shown above. +3. Add the scheme's `.F90` to your build's source list. + +--- + +## 3. Host Fortran requirements + +### 3.1 Required control variables + +Every host's `type=control` table must declare: + +| Standard name | Fortran type | Purpose | +|-----------------------------------|--------------|-----------------------------------| +| `suite_name` | character | Drives suite dispatch | +| `horizontal_loop_begin` | integer | Lower chunk-bound | +| `horizontal_loop_end` | integer | Upper chunk-bound | +| `number_of_physics_threads` | integer | Physics-internal budget | +| `ccpp_error_code` | integer | Error flag | +| `ccpp_error_message` | character | Error message | + +Paired-optional — two symmetric `(index, count)` pairs (see §1.7). +For each pair, declare **both** members in `type=control` or +**neither**; declaring exactly one is a hard error: + +| Standard name | Fortran type | Table type | Purpose | +|-------------------------|--------------|------------|--------------------------------| +| `instance_number` | integer | control | Current instance index | +| `number_of_instances` | integer | control | Total instance count | +| `thread_number` | integer | control | Current thread / per-thread-container index | +| `number_of_threads` | integer | control | Total thread count | + +**The `type=control` table is a closed set.** It may contain *only* +the variables in the two tables above (the 7 required plus the 4 +paired-optional pair members — 11 standard names total). Any other +variable in a `type=control` table is a hard error: a host quantity +that schemes consume belongs in a `type=host` table, and the subcycle +loop variables (`ccpp_loop_counter` / `ccpp_loop_extent`) are +generator-owned locals you never declare. + +### 3.2 Required entry-point call sequence + +``` +ccpp_register(suite_name, errflg, errmsg, [instance_number, number_of_instances]) + └── per scheme that declares a register phase +ccpp_init(suite_name, errflg, errmsg, [instance_number, number_of_instances]) + └── per scheme that declares an init phase +ccpp_physics_init(...) + └── physics phase routines per group: + ccpp_physics_init + ccpp_physics_timestep_init + ccpp_physics_run ← run-loop phase + ccpp_physics_timestep_final + ccpp_physics_final +ccpp_final(suite_name, errflg, errmsg, [instance_number, number_of_instances]) +``` + +The `(instance_number, number_of_instances)` pair appears in every +signature only when the host declares it (§1.7). Both flow uniformly +through lifecycle and physics-phase calls; the framework consumes +`number_of_instances` only at register/init time but carries it +elsewhere for API symmetry. + +### 3.3 Module-name convention (host, scheme, and DDT tables) + +capgen trusts metadata and does **not** parse Fortran, so it derives +the Fortran module name from the metadata: by default `module name = +table name`. When the Fortran `module` statement does not match the +`[ccpp-table-properties] name`, declare the real module name with the +`module_name` override (§1.2). This applies to **every** table type: + +``` +[ccpp-table-properties] + name = test_host_data + type = host + module_name = mod_test_host_data +``` + +The same rule bites **scheme** tables. If a scheme file's table is named +`gravity_wave_drag_common` but the Fortran is `module gw_common`, the +generated cap emits `use gravity_wave_drag_common` and the build fails +with `Cannot open module file 'gravity_wave_drag_common.mod'`. Fix it in +the scheme `.meta`: + +``` +[ccpp-table-properties] + name = gravity_wave_drag_common + type = scheme + module_name = gw_common +``` + +Standalone `type = ddt` tables **require** `module_name` explicitly +(there is no basename fallback). Porting a host like CAM-SIMA's +`atmospheric_physics` tree, where many scheme/DDT table names differ from +their module names, is largely a batch of `module_name` injections. + +### 3.4 Registered scalar-index dimensions + +A small set of CCPP standard-name dimensions are *registered*: each +one is a count that capgen auto-collapses to a paired scalar index +variable at every access site. + +| Count dim (in `dimensions = (...)`) | Index var (capgen substitutes) | +|---|---| +| `number_of_instances` | `instance_number` | +| `number_of_threads` | `thread_number` | + +**Where these may appear**: ONLY on container DDT-instance variables in +the access path. Example: + +``` +[Interstitial] + standard_name = GFS_interstitial_type_instance + type = GFS_interstitial_type + dimensions = (number_of_threads) +``` + +Every scheme that reaches into `Interstitial%` will see the +generator emit `physics%Interstitial(thread_number)%` at the +call site — no metadata work required on the scheme side. + +**Two rules govern this:** + +1. *(generalized)* A container DDT-instance variable may carry any + registered scalar-index dim — single (`(number_of_threads)`) or + paired (`(number_of_instances, number_of_threads)`). Dims that + AREN'T registered flow through the normal slice machinery + (`horizontal_loop_begin:horizontal_loop_end`, `1:vertical_*`, …) + just like flat-array dims. +2. *(enforced — hard parse-time error)* A **leaf** variable + (intrinsic-typed or `external:` — the kind a scheme binds to) + **MUST NOT** declare a registered scalar-index dim. If you write:: + + [my_array] + type = real | kind = kind_phys + dimensions = (number_of_threads, horizontal_dimension) # ILLEGAL + + capgen will reject it at parse time with a message pointing + at the wrap-in-DDT remediation pattern. Wrap the leaf in a + container DDT instead. + +The registered table lives in +[`capgen/metadata/registered_dimensions.py`](../capgen/metadata/registered_dimensions.py). +It carries a four-step recipe at the top of the file for adding new +pairings. + +--- + +## 4. Generator CLI and build integration + +### 4.1 `ccpp_capgen.py` invocation + +``` +python ccpp_capgen.py \ + --host-files [,,...] \ + --scheme-files [,,...] \ + --suites [,,...] \ + --host-name \ + --output-root /ccpp \ + [--kind-type =[:]] \ + [--legacy-mode] \ + [--gfs-dim-aliases] \ + [--legacy-auto-clone-constituents] \ + [--no-host-introspection] \ + [--verbose] [--verbose] +``` + +`--kind-type` syntax: `=[:]`. When `:` is +omitted, `` must be an ISO_FORTRAN_ENV constant (REAL32/REAL64/...) +and the module defaults to `iso_fortran_env`. `kind_phys` is +auto-defaulted to `iso_fortran_env:REAL64` when not supplied. + +#### Transient migration shims + +Three opt-in flags exist for migrating legacy hosts. Each is +self-contained and grep-tagged for clean removal: + +**`--legacy-mode`** (transient migration shim, will be removed): +silently rewrites a small set of deprecated CCPP standard names to +their capgen equivalents at parse time — see §1.8 for the full +table (`horizontal_loop_extent` → `horizontal_dimension`, +`number_of_openmp_threads` → `number_of_threads`). The rewrite fires +for both standard-name attributes AND dimension tokens. Prints a +loud warning banner at startup, enumerating every pair the shim is +rewriting, so the substitution is never invisible. Available on both +`ccpp_capgen.py` and `ccpp_validator.py` (keep the flag consistent +between the two when both are invoked from CMake). All translation +logic is isolated in `metadata/legacy_compat.py` and tagged with +`# legacy-compat:` comments at every touchpoint. + +**`--gfs-dim-aliases`** (transient migration shim, see §1.10): +treats GFS-physics names +`adjusted_vertical_layer_dimension_for_radiation` and +`vertical_composition_dimension` as equivalent to +`vertical_layer_dimension` **inside the resolver's per-position +dim-identity check only** (upper bound only). The host variables +themselves stay distinct. Generator-only (the validator never +reaches the dim canonicaliser). Module `metadata/dim_aliases.py`; +touchpoints tagged `# dim-aliases:`. + +**`--legacy-auto-clone-constituents`** (transient migration shim, see +`doc/auto_clone_constituents.md` for the full reference): +reinstates original ccpp-capgen's auto-clone-static-constituent +registration path. Every `is_constituent` consumer scheme arg +(`advected = True`, `constituent = True`, or `molar_mass = …`) with +no register-phase source is auto-registered into the per-suite +dynamic-constituents buffer using values lifted straight from the +scheme metadata (with sensible defaults: `long_name` synthesised from +the standard name when missing, `diag_name` falls back to local_name, +`vertical_dim` lifted from the arg's dim list). Adds four legacy +`%instantiate` kwargs to the parser (`default_value`, `min_value`, +`water_species`, `mixing_ratio_type`). Available on both +`ccpp_capgen.py` and `ccpp_validator.py` (the validator must +accept the four extra attrs). **Single-instance only** — declaring +the `instance_number` + `number_of_instances` pair while the flag is +on is a hard error before any suite is parsed. Module +`metadata/auto_clone_constituents.py`; touchpoints tagged +`# auto-clone-constituents:`. + +### 4.2 `ccpp_datafile.py` query CLI + +Generated `datatable.xml` carries: + +- `` — generated outputs (utilities/host_files/suite_files). +- `` — `.meta` and expanded SDF. +- `` — per-scheme call lists, **scoped to schemes that are + actually referenced by the loaded suites** (group phase calls + the + suite-level ``/`` hooks). Scheme metadata files passed + on the CLI but never referenced are silently dropped. +- `` — `dependencies = …` from host/control/ddt tables + (always) plus the same per-scheme list as `` (filtered to + the used set). Build systems that compile against + `ccpp_datafile.py --dependencies` therefore only pull in scheme deps + for compiled schemes; missing transitive deps in scheme metadata + surface as link errors and should be fixed in the `.meta` file. +- `` — host/api/suite/group dictionaries. + +Query via `ccpp_datafile.py -- `. Flags include +`--dependencies`, `--capgen-files`, `--host-files`, `--utility-files`, +`--suite-files`, `--scheme-files`, `--suite-list`, +`--required-variables `, `--input-variables `, +`--output-variables `, `--host-variables`, `--show`. + +`--suite-files` returns capgen-generated cap files (`ccpp__cap.F90`, +etc.). `--scheme-files` returns the **user-supplied scheme `.F90` sources** +that the loaded suites actually reference — the filtered compile manifest. +Each used scheme's source is resolved as `/.` +(extension preference order: `.F90`, `.f90`, `.F`, `.f`); missing files are +warned about and the canonical `.F90` guess is emitted so the build-system +query stays useful. + +### 4.3 CMake helpers + +`cmake/ccpp_capgen.cmake` and `cmake/ccpp_validator.cmake` provide the +`ccpp_capgen(...)` and `ccpp_validator(...)` macros. `ccpp_datafile(...)` +queries datatable.xml at configure time. + +### 4.4 No-op regeneration preserves mtimes + +Every generated file (caps, `datatable.xml`, `ccpp_kinds.F90`, expanded +SDFs, `.meta` artifacts) goes through `write_if_changed`: the new content +is staged to a sibling temp file under the output root and atomically +replaces the target only when the bytes actually differ. Reruns with +identical inputs therefore leave on-disk mtimes untouched, so CMake / +Make / Ninja do not trigger a downstream rebuild cascade. Matches the +behavior of legacy `ccpp-prebuild` / `ccpp-capgen`. The staging temp +file lives in the target's parent directory (always under +`--output-root`), so no `/tmp` access is required. + +### 4.5 Driving an existing capgen-based build: the CAM-SIMA compatibility layer + +A host whose build system was written against **original ccpp-capgen's +Python API** can adopt capgen without rewriting that build system, by +inserting a thin facade. CAM-SIMA does exactly this with +`cime_config/capgen_compat/` (in the CAM-SIMA tree, not in capgen). +CAM-SIMA's `cam_autogen.py`, `generate_registry_data.py`, and +`write_init_files.py` are unmodified; they import the facade instead of +original capgen and keep calling the same object surface +(`cap_database.host_model_dict()`, `cap_database.call_list(phase)`, +`Var.get_prop_value(...)`, `Var.source.ptype`, …). + +The facade re-implements that surface on top of capgen's outputs: + +- `_runner.py` invokes `ccpp_capgen.py` and returns the resolver + results plus the `datatable.xml`. +- `_cap_database.py` (`CapDatabase`) exposes `host_model_dict()` over the + flat `host_dict` and `call_list(phase)` over the per-(scheme, phase) + `ResolvedArg` lists, mapping original-capgen phase spellings + (`initialize`/`finalize`) onto capgen's (`init`/`final`). +- `_var_wrapper.py` (`_VarWrapper`) reconstructs original capgen's + per-variable accessors over a `HostVarEntry` (host path) or a + `ResolvedArg` (call-list path). +- `metadata_table.py` / `parse_*` shim the metadata-parsing entry points + the registry generator expects. + +Two contracts matter when writing or maintaining such an adapter, both +learned from the CAM-SIMA bring-up: + +1. **Drop suite-internal args.** A `ResolvedArg` with + `source == 'suite'` is produced by one scheme and consumed by another + within the same suite (it lives in `_data`, never in the host + dict). The adapter must NOT surface it on the call list, or the host's + init-file generator will mis-flag it as a "missing required host + variable". +2. **Key constituent handling on the source.** Treat a `ResolvedArg` + with `source == 'constituent'` as supplied by the constituents object + (skip host USE-import and skip the initial-conditions read). Do **not** + key on `ResolvedArg.is_constituent`: that flag reflects whether the + *scheme* flagged the arg, and an unflagged rule-b consumer (§6.5) of a + constituent or `tendency_of_` carries `is_constituent = False` while + still being framework-supplied. Getting this wrong was the `se_cslam` + "Missing required host variables: tendency_of_water_vapor_…" failure + (`doc/constituents_overhaul.md` §4.15). + +This facade is how capgen currently drives the `kessler`, `rrtmgp`, +and `se_cslam`/CSLAM (FCAM7 `cam7`) CAM-SIMA cases end-to-end on Derecho +— building and running to completion under both **gnu and intel**, with +bit-comparable results. A short shareable brief (for the original +ccpp-capgen author) is `doc/capgen_compat_layer.md`; the full developer +reference is `cime_config/capgen_compat/README.md` in the CAM-SIMA tree. + +--- + +## 5. Generated cap layout — what's new and what changed + +### 5.1 Output files + +Always generated: + +- `ccpp_kinds.F90` — kind parameters. Listed under ``. +- `_ccpp_cap.F90` — public host-facing entry points + introspection routines. + Filename and emitted `module _ccpp_cap` name are both driven by the + required `--host-name ` CLI argument so multiple host integrations + can co-exist in one executable. The public sub names inside + (`ccpp_register`, `ccpp_init`, `ccpp_physics_*`, `ccpp_final`) are + unchanged regardless of ``. +- `ccpp__cap.F90` — per-suite dispatcher. +- `ccpp___cap.F90` — per-group phase implementations. +- `ccpp__data.F90` — suite-owned interstitial DDT + module-level array. +- `ccpp__types.F90` — pointer-wrapper types for optional args. +- `ccpp__data.meta` — inspection artifact; pairs with `ccpp__data.F90` (`.meta` ↔ `.F90` filename convention). +- `datatable.xml` — build-system + host-introspection metadata. + +When any scheme registers constituents: + +- `ccpp_host_constituents.F90` — owns `ccpp_model_constituents_obj(:)` + and the host-facing constituent API. + +### 5.2 Per-suite data: TARGET on the instance array + +`ccpp_suite_data(:)` carries the `TARGET` attribute: + +```fortran +type(ccpp__data_t), allocatable, target, public :: ccpp_suite_data(:) +``` + +This makes every `ccpp_suite_data(i)%component(...)` subobject a valid +pointer-assignment target — needed for transformation temps and +optional-arg pointer wrappers. + +### 5.3 Variable transformations + +The generator emits three kinds of transform on a per-arg basis: + +| Transform | Trigger | +|------------------|--------------------------------------------------| +| Unit conversion | `host.units != scheme.units` with a registered conversion entry. | +| Kind conversion | `host.kind != scheme.kind` (different strings). | +| Vertical flip | `host.top_at_one != scheme.top_at_one` on a var with a vertical dim. | + +Transforms only smooth over *representation* differences. Anything +the cap cannot bridge with a per-call copy — type identity, rank, +or per-position dimension identity — is rejected by the resolver as +a hard error (see §1.3.2). In particular, the kind-conversion entry +above is *not* gated on convertibility: any kind-string difference +triggers an implicit conversion copy. Watch for unintended +narrowing. + +These compose. A scheme arg that needs unit + flip emits a single +combined assignment through a transformation temp: + +```fortran +temp_l = 1.0E-3_kind_phys*host_var(lb:ub, nlev:1:-1) ! unit conversion: kind_phys to kind_phys; vertical flip (top_at_one mismatch) +call scheme_run(temp=temp_l, ...) +host_var(lb:ub, nlev:1:-1) = 1.0E+3_kind_phys*temp_l ! ... reverse ... +``` + +Identity unit conversions (registered for dimensionally-equivalent +spellings like `J kg-1 ↔ m2 s-2`, formula `'{var}'`) are not labeled +"unit conversion" in the comment. + +### 5.4 Subcycle emission + +```fortran +integer :: ccpp_loop_counter +integer :: ccpp_loop_counter_2 +... +do ccpp_loop_counter = 1, phys_state%num_subcycles ! outer + call scheme_pre(...) + do ccpp_loop_counter_2 = 1, 2 ! inner + call scheme_calc(...) + end do +end do +``` + +### 5.5 State machine + +Per-instance integer state arrays: + +- `ccpp_suite_state(:)` — suite-level (UNREGISTERED / REGISTERED / + FRAMEWORK_INITIALIZED). +- `ccpp_group_state(:)` — group-level (UNINITIALIZED / INITIALIZED / + IN_TIMESTEP). + +Single-instance hosts get length-1 arrays indexed with literal `1`. +See `doc/redesign_prompt.md` §7. + +**Idempotent entry points.** `ccpp_physics_init`, `ccpp_physics_final`, and +`ccpp_final` are all silently idempotent — repeat calls return cleanly with +`errflg=0` rather than erroring. `ccpp_physics_final` additionally silent-skips +when issued *after* `ccpp_final` has torn the suite down (state array +deallocated on the last instance, or `== UNREGISTERED` on any other instance). +The other physics phases (`timestep_init`, `run`, `timestep_final`) still +hard-error on a state mismatch. `ccpp_init` does *not* silent-skip when the +state array is unallocated — there, "not allocated" really does mean +"you forgot `ccpp_register`" and continues to be a hard error. + +--- + +## 6. Framework changes (constituents) + +### 6.1 `ccpp_constituent_prop_mod` ownership flag + +(Framework PR — needs upstream merge.) Adds: + +- `framework_owns_me` private flag on `ccpp_constituent_properties_t`, + default `.false.`. +- `set_framework_owned(value)` setter (call before + `obj%new_field(const_prop, ...)` when transferring ownership). +- `is_framework_owned()` getter. +- `ccpt_deallocate` only frees when the flag is set; otherwise just + nullifies. + +Backward-compatible. Original capgen's auto-clone path in +`scripts/constituents.py` has been updated to call the setter. +capgen's `--legacy-auto-clone-constituents` shim (§6.4) +synthesises `%instantiate(...)` directly on slots of the per-suite +dynamic-constituents buffer, so the properties objects are owned by +the buffer from creation — no ownership transfer call needed. + +### 6.2 capgen constituent API + +(See `doc/constituents.md` for the full reference.) Highlights: + +- One `ccpp_model_constituents_obj(:)` array per generator invocation, + sized to `number_of_instances`. +- Host-facing API: + - `ccpp_register_constituents(host_constituents, instance_number, ...)` + - `ccpp_initialize_constituents(ncols, num_layers, instance_number, ...)` + - `ccpp_const_get_index(stdname, const_index, instance_number, ...)` + - `ccpp_constituents_array(instance_number) → pointer` + - `ccpp_advected_constituents_array(instance_number) → pointer` + - `ccpp_model_const_properties(instance_number) → pointer` + - `ccpp_number_constituents(num_flds, advected, instance_number, ...)` + - `ccpp_gather_constituents`, `ccpp_update_constituents` + - `ccpp_is_scheme_constituent(var_name, ...)` (not per-instance) +- Scheme-side registration rules — register-phase + `ccpp_constituent_properties_t(:)` arg declares new constituents; + flag a base species with `advected=true intent=in/inout`; produce a + tendency with `constituent=true intent=out` + `tendency_of_` std + name; a constituent-flagged `intent=out` that is not a `tendency_of_*` + is a codegen error. A scheme that only READS a constituent or a + `tendency_of_` need not re-flag it — see §6.5. +- **`_register` is called exactly once per scheme** (2026-06-08). + capgen packs each constituent scheme's returned + `ccpp_constituent_properties_t(:)` array into the per-suite buffer in a + single append pass, so a register routine may safely allocate persistent + module state. (An earlier two-pass count+copy called register twice and + broke any non-idempotent register, e.g. `prescribed_aerosols_register` + allocating a module-level map.) + +### 6.3 Host metadata wins over auto-provisioning (2026-05-12) + +If the host declares a framework-named standard name +(`ccpp_constituents` / `ccpp_constituent_tendencies` / +`ccpp_constituent_properties` / `number_of_ccpp_constituents` / +`index_of_`) as a regular host variable, the resolver uses the +host's declaration and skips capgen auto-provisioning. Matters +most for legacy hosts (GFS / SCM) that own their own tracer +indices — e.g. `[ntcw]` with `standard_name = +index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array` +resolves to the host's short local name `ntcw`, not a parallel +module-level integer named after the full standard name (which +would also blow Fortran's 63-char identifier limit). See +`doc/constituents.md` §3. + +Active design review for the next constituents iteration: +`doc/constituents_overhaul.md` (Class A vs Class B property +classification, three reform proposals). + +### 6.4 Legacy auto-clone registration (`--legacy-auto-clone-constituents`) + +For hosts that ship metadata in original ccpp-capgen's shape — most +notably CAM-SIMA's atmospheric_physics tree, where ~16 of the ~20 +constituent-touching schemes declare `advected = True` (or +`constituent = True`, or `molar_mass = …`) in `_run` arg tables and +rely on the framework to register the constituent — pass +`--legacy-auto-clone-constituents` to both `ccpp_capgen.py` and +`ccpp_validator.py`. + +What changes: + +- The parser accepts four extra scheme-arg attributes + (`default_value`, `min_value`, `water_species`, + `mixing_ratio_type`). Fortran-style literal suffixes + (`0.0_kind_phys`, `1.0d-5`, `-3.14_8`) are accepted on the real + fields, since legacy metadata writes the values in source form. +- For every unique standard name that appears as an `is_constituent` + consumer with no register-phase source, the suite cap emits a + synthesised `%instantiate(...)` call into the per-suite + dynamic-constituents buffer. The scheme author writes no Fortran + registration code. +- `long_name` is auto-synthesised from the standard name when missing + (`cloud_liquid_dry_mixing_ratio` → `'Cloud liquid dry mixing + ratio'`); `diag_name` falls back to local_name; `vertical_dim` is + lifted from the arg's `dimensions = (...)` entry. +- Schemes that pass the whole constituents buffer (e.g. + `apply_constituent_tendencies_run` with `ccpp_constituents` / + `ccpp_constituent_tendencies` / `index_of_*` args) are excluded + from auto-clone — those resolve through the framework + whole-buffer path, not as individual registrations. + +What capgen's other rules still require (the shim does **not** +relax them): + +- `intent = inout` on base constituents (`advected = True` on a + non-`tendency_of_*` std_name). `intent = out` is reserved for + tendency args. +- Metadata arg tables must match the Fortran subroutine signature. + Declaring a constituent in `_init`'s arg table when + `_init` doesn't take it as a Fortran dummy is rejected by + the validator. + +Single-instance only: declaring `instance_number` + +`number_of_instances` while the flag is on aborts before any suite is +parsed. Legacy hosts predate multi-instance support, so this matches +the use case. + +Full reference: `doc/auto_clone_constituents.md`. E2e fixture: +`end-to-end-tests/advection_auto_clone/` (a port of CAM-SIMA's +`advection_test`). + +### 6.5 Reading constituents and tendencies without re-flagging (rule b, 2026-06-05) + +Whether a given standard name is a constituent or an ordinary variable +is the **host's** decision: CAM-SIMA exposes water vapor as a +constituent; CCPP-SCM may expose the same name as an ordinary host +variable. A scheme that merely **reads** such a name therefore must +**not** repeat the `advected` / `constituent` flag — only the +declaring/producing scheme (or the host) does. + +capgen infers constituent-ness for an unflagged consumer from the +scheme-metadata-wide set of names that *some* scheme flags +(`VariableResolver.constituent_stdnames()`): + +- an unflagged `intent=in/inout` read of a flagged base name resolves to + `…%vars_layer(:, …, index_of_)`; +- an unflagged `intent=in` read of `tendency_of_` resolves to + `…%vars_layer_tend(:, …, index_of_)` — the same column a constituent + tendency *producer* wrote. + +**Host / earlier-suite provision wins**: if the host declares the name, +or an earlier scheme already produced it as an ordinary variable, normal +host/suite resolution takes over and no constituent column is used. + +This is what lets the CAM-SIMA `cam7` suite work unchanged: the +convection/stratiform schemes write `tendency_of_water_vapor_…` as a +flagged constituent tendency and the `sima_diagnostics` schemes read it +back unflagged. Host adapters that post-process the resolver output +(e.g. CAM-SIMA's `write_init_files` via the compatibility layer, §4.5) +must key constituent handling on `ResolvedArg.source == 'constituent'`, +**not** on `ResolvedArg.is_constituent` — an inferred consumer carries +`is_constituent = False` by design. + +### 6.6 `number_of_ccpp_constituents` as a dimension + +A scheme (or a suite-owned interstitial) may be dimensioned by the +framework constituent count `number_of_ccpp_constituents`. capgen +resolves that count for *any* variable: call-site subscripts emit `:` +for the constituent axis, and `_data` allocations size the axis +from the per-instance constituent object's `%num_layer_vars`. This is +what allows whole-buffer schemes (e.g. constituent advection) to declare +`dimensions = (horizontal_dimension, vertical_layer_dimension, +number_of_ccpp_constituents)`. E2e fixture: +`end-to-end-tests/constituents_dim/`. + +--- + +## 7. Validator + +`capgen/ccpp_validator.py` — standalone Fortran-vs-metadata checker. +Validates **scheme** metadata against scheme Fortran files, and (since +2026-06-01) **host** and **DDT** metadata against host module-level +declarations and derived-type definitions. + +### 7.1 What the validator checks + +For every `(scheme, phase)` declared in the supplied `.meta` files: + +1. The Fortran subroutine `_` **exists** in the source + tree (auto-discovered via `source_path` on the table, or supplied + explicitly with `--source-files`). +2. **Argument count** — the number of dummy arguments matches the + metadata, after subtracting any optional-only-in-Fortran args (see + §7.2). +3. **Argument names** — the set of metadata `local_name` values + matches the Fortran dummy-arg list (order-insensitive, + case-insensitive). +4. For every argument present on **both sides**, per-attribute + consistency: + + | Attribute | Behavior | + |------------|----------| + | `intent` | Strict match (`in` / `out` / `inout`). Metadata declares it but Fortran omits → error. | + | `type` | Case-insensitive match. `double precision` / `doubleprecision` / `double precision` are normalized to the same form. DDT names match the Fortran `type(name)` / `class(name)` wrapper — metadata `type = ty_rad_lw` matches Fortran `type(ty_rad_lw)`. External types match by typename — metadata `type = external:mpi_f08:mpi_comm` matches Fortran `type(mpi_comm)` (the module qualifier is metadata-only). | + | `kind` | Case-insensitive match. **Character length must be CONSISTENT** — the metadata mirrors the Fortran exactly: `len=*` matches only `len=*`, and `len=N` only the identical `len=N` (no wildcarding; changed 2026-06-08 — the validator runs first, so a Fortran `len=*` can only pair a metadata `len=*`, and vice versa). Old-style F77 forms (`character*64`, `character*(*)`, per-entity `c*5` / `d(10)*8`) are normalized to the `len=` form before comparison. | + | `rank` | Number of dimensions only. Reads both `dimension(...)` line attributes and var-attached `foo(:,:)` syntax. Per-dimension bound comparison is NOT done. | + +### 7.2 Asymmetric `optional` rule + +| Metadata | Fortran | Outcome | +|-----------------|--------------------------|----------| +| (absent) | `optional` | warning | +| (absent) | required (no `optional`) | error | +| `optional=False`| `optional` | warning | +| `optional=False`| required (no `optional`) | OK | +| `optional=True` | `optional` | OK | +| `optional=True` | required (no `optional`) | **error**| + +Reason for the asymmetry: a metadata-side `optional=True` is a promise +to the cap that the value may be absent at the call site. If Fortran +requires the dummy, the cap's `present()` check is invalid. The +reverse direction (Fortran allows optional, metadata always passes it) +is a valid subset of the Fortran contract — the arg is always present +and any optional Fortran dummy can accept that — so we warn but don't +fail the build. + +### 7.3 Continuation-line handling + +Covers both free-form (`&` at trailing end of prior line only) and +fixed-form / dual-form (`&` at both ends, with the leading marker at +column 6). Comment-only and blank lines interleaved between +continuation lines are skipped as Fortran 90+ permits. When the +signature parser finds a subroutine but extracts zero args while +metadata declares many, the "Argument count mismatch" error appends a +HINT pointing at the parser rather than masquerading as a real +mismatch — common cause is an unsupported signature feature. + +### 7.4 Host and DDT metadata validation (2026-06-01) + +Pass `--host-files` to validate `type = host` and `type = ddt` tables +against module-level declarations and derived-type definitions in the +same `--source-files` Fortran tree: + +``` +ccpp_validator.py \ + --scheme-files scheme1.meta,scheme2.meta \ + --host-files host.meta,physics_types.meta \ + --source-files scheme1.F90,scheme2.F90,host.F90,physics_types.F90 +``` + +Per-table behaviour: + +| Table type | Check | +|---|---| +| `type = host` | For each variable, find a module-level declaration of the same `local_name` (lowercased; subscripted spellings like `tk(:,:)` strip to `tk`) in the Fortran module named by `module_name` (or `table_name` when not overridden). Compare type / kind / rank using the same rules as the scheme-side check (intent is silently ignored — host vars carry none). Missing module or missing variable → clear error. | +| `type = ddt` | For each component, find a matching member of the Fortran derived type whose name equals `table_name` (the DDT name). The type definition may live in any parsed module (a flat cross-file index is built from `--source-files`). Same type / kind / rank rules as host vars. | +| `type = control` | Silent skip with an INFO log line — control vars are framework-injected at the cap call sites, no host Fortran backs them. | +| `type = scheme` in `--host-files` | Hard error — schemes must be passed via `--scheme-files` so the validator can find the per-phase subroutines. | +| `type = host` / `control` / `suite` in `--scheme-files` | Hard error — symmetric to the rule above. Misclassified `.meta` files fail fast with a pointer at the correct flag. **`type = ddt` is allowed in `--scheme-files`**: schemes routinely co-locate their own derived-type definitions (e.g. radiation schemes carrying `ty_rad_lw` / `ty_rad_sw` in the same `.meta` as the scheme phase blocks). Scheme-co-located DDTs go through the same per-component validation as host-side DDTs. | + +**Inputs contract.** At least one of `--scheme-files` or `--host-files` +must be supplied. Passing neither raises a clear error rather than +the older silent "Validation passed." Either flag alone is fine; +both together is the common case. + +The same per-attribute rules apply that the scheme-side check uses, +which is one rule fewer than the scheme side: there is no `optional` +flag on module vars / DDT components, and host metadata carries no +`intent`, so the asymmetric-optional rule (§7.2) does not apply here. +Character length is matched exactly (§7.1). Additionally, because +host / DDT metadata *defines* its character storage (a module variable +or a derived-type component, neither of which may be assumed-length), +`len=*` is rejected there outright — host and DDT character variables +must declare a concrete `len=N`. Assumed length is valid only for +dummy arguments (scheme args and control/lifecycle variables). + +--- + +## 8. Known gaps and deferred items + +| Item | Status | +|--------------------------------------------|-----------------------------------------------| +| `ccpp_loop_counter` standard name inside nested subcycles | Maps to OUTERMOST loop var. None of cam-sima uses this; revisit if a scheme needs the innermost value. | +| Validator host-metadata check | **Landed 2026-06-01**: pass `--host-files`; see §7.4. | +| Constituents overhaul (Class A/B + setters) | Discussion doc at `doc/constituents_overhaul.md`. | +| Framework setters: `set_advected`, `set_diagnostic_name`, `set_default_value` | Deferred; depends on constituents-overhaul decision. | +| Codegen-time scheme-registration cross-check | Deferred; would require new `registers_std_names` metadata attr. | +| `_FRAMEWORK_CONST_DIM_INPUTS` cleanup | **Done 2026-05-13**: hand-curated frozenset gone; framework-constituent dim refs ride on a dedicated `used_const_dim_std_names` field on `ResolvedArg`. | +| Suppress `ccpp_host_constituents.F90` when unused | Deferred; currently emitted for every build even when no scheme/host actually exercises the constituent system. Now *correct* (empty) for SCM-style hosts thanks to the host-wins rule, but still dead code. See `design_constituent_host_wins.md`. | +| Python linter / formatter pass | Deferred; pick `ruff` and apply across `capgen/`. | +| Generated Fortran ↔ Codee formatter idempotency | Deferred; emitted `.F90` must round-trip cleanly through the project's Codee Fortran formatter. | +| `fortran_to_metadata` developer utility | Deferred; bootstraps a `.meta` skeleton from an existing `.F90` subroutine. | +| `--legacy-mode` shim removal | Transient; remove `metadata/legacy_compat.py`, `unit-tests/test_legacy_compat.py`, and every `# legacy-compat:` touchpoint when scheme metadata has migrated. | +| `--gfs-dim-aliases` shim removal | Transient; remove `metadata/dim_aliases.py`, `unit-tests/test_dim_aliases.py`, and every `# dim-aliases:` touchpoint when GFS metadata stops spelling `vertical_layer_dimension` as `adjusted_vertical_layer_dimension_for_radiation` / `vertical_composition_dimension`. | +| `--legacy-auto-clone-constituents` shim removal | Transient; remove `metadata/auto_clone_constituents.py`, `unit-tests/test_auto_clone_constituents.py`, sample files under `unit-tests/sample_files/scheme_auto_clone_consumer.meta` + `sample_suite_files/suite_auto_clone.xml`, and every `# auto-clone-constituents:` touchpoint when consumers have moved to explicit `host_constituents(:)` declaration or register-phase scheme registration. | +| `ccpp_datafile.py` query CLI rework | Deferred (2026-05-13); collapse `--host-files` / `--suite-files` / `--utility-files` into `--capgen-files`, then repurpose `--host-files` as a filtered list of **input** host metadata files (parallel to `--scheme-files`). Most hosts pack all host data into a handful of shared files, so the filtering pay-off is small — the draw is API symmetry. | + +--- + +## Cross-references + +- `doc/redesign_prompt.md` — original design specification (sections + marked "historic" where the implementation has evolved). +- `doc/redesign_analysis.md` — analysis of the legacy ccpp-prebuild + + ccpp-capgen toolchains. +- `doc/constituents.md` — full constituents reference for capgen. +- `doc/constituents_overhaul.md` — architecture review and reform + proposals for the next iteration. +- `doc/capgen_compat_layer.md` — short brief on the CAM-SIMA ↔ capgen + compatibility layer (§4.5); full reference is + `cime_config/capgen_compat/README.md` in the CAM-SIMA tree. + diff --git a/doc/redesign_analysis.md b/doc/redesign_analysis.md new file mode 100644 index 00000000..cb8e4d7a --- /dev/null +++ b/doc/redesign_analysis.md @@ -0,0 +1,2674 @@ +# CCPP Framework Code Generator — Technical Analysis for Redesign + +*Analysis date: 2026-05-04. Clarifications added: 2026-05-05.* + +This document is a deep-dive technical analysis of the two existing CCPP Framework code generators — +`ccpp-prebuild` and `ccpp-capgen` — produced as input to a planned complete redesign. +It covers execution flow, data structures, feature sets, build system integration, and +key architectural differences. + +--- + +## Table of Contents + +1. [Background and motivation](#1-background-and-motivation) +2. [ccpp-prebuild — detailed analysis](#2-ccpp-prebuild--detailed-analysis) +3. [ccpp-capgen — detailed analysis](#3-ccpp-capgen--detailed-analysis) +4. [Shared infrastructure](#4-shared-infrastructure) +5. [Feature comparison](#5-feature-comparison) +6. [Build system integration](#6-build-system-integration) +7. [Key architectural differences](#7-key-architectural-differences) +8. [Design considerations for the redesign](#8-design-considerations-for-the-redesign) +9. [Real-world example: CCPP Single Column Model (SCM)](#9-real-world-example-ccpp-single-column-model-scm) +10. [Real-world example: CAM-SIMA (capgen)](#10-real-world-example-cam-sima-capgen) +11. [Real-world example: UFS Weather Model (prebuild)](#11-real-world-example-ufs-weather-model-prebuild) +12. [Real-world example: Navy NEPTUNE (prebuild, restricted)](#12-real-world-example-navy-neptune-prebuild-restricted) +13. [Cross-cutting design decision: how host data enters the cap chain](#13-cross-cutting-design-decision-how-host-data-enters-the-cap-chain) + +--- + +## 1. Background and motivation + +The CCPP Framework is a code generator that analyzes metadata describing variables required +by physical parameterizations in numerical weather prediction (NWP) models, compares them +against metadata provided by a host model, and generates Fortran interface ("cap") code that +connects the two. + +There are two generations of the generator: + +**`ccpp-prebuild`** (`scripts/ccpp_prebuild.py`): +- Simple, mostly procedural Python +- Used in: NOAA UFS Weather Model, Navy NEPTUNE, CCPP-SCM +- Extremely reliable in research, development, and operations +- Fewer capabilities; simpler design + +**`ccpp-capgen`** (`scripts/ccpp_capgen.py`): +- Highly complex, object-oriented Python taken to the extreme +- Used in: NCAR CAM-SIMA (still mostly a research/development model) +- Many advanced features designed but never implemented (funding/priority gaps) +- Notoriously difficult to develop; no remaining team member fully understands it + +**The original plan** was to update `ccpp-capgen` with missing features from `ccpp-prebuild` +and transition all models to it. **This plan has been abandoned** in favor of a complete +redesign that draws the best lessons from both generations. + +The immediate trigger for abandoning capgen was the failure — after considerable effort by +three developers — to make capgen pass DDT arguments to group caps the way prebuild does. +This is the root cause of capgen's severe performance problem (seconds for prebuild, +10+ minutes for capgen on the same suite set) and of its broken handling of optional +variables under Fortran compiler debugging flags. + +--- + +## 2. ccpp-prebuild — detailed analysis + +### 2.1 Command-line arguments and configuration + +Entry point: `scripts/ccpp_prebuild.py`, `main()`. + +Arguments parsed by `argparse`: + +| Argument | Required | Purpose | +|---|---|---| +| `--config` | yes | Path to host-model Python config module | +| `--suites` | no | Comma-separated suite names (without `.xml`) | +| `--builddir` | no | Override build directory from config | +| `--namespace` | no | Appended to static API module name | +| `--debug` | no | Insert Fortran array-size checks in generated caps | +| `--clean` | no | Remove generated files and exit | +| `--verbose` | no | Set logging to DEBUG | + +The `--config` file is a plain Python module imported dynamically via `importlib`. +Key variables it must define: + +| Config variable | Purpose | +|---|---| +| `VARIABLE_DEFINITION_FILES` | List of host-model Fortran sources with metadata hooks | +| `SCHEME_FILES` | List of physics scheme Fortran sources | +| `CAPS_DIR` | Output directory for generated cap `.F90` files | +| `SUITES_DIR` | Directory containing suite definition XML files | +| `STATIC_API_DIR` | Output directory for `ccpp_static_api.F90` | +| `TYPEDEFS_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for typedef build snippets | +| `SCHEMES_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for scheme build snippets | +| `CAPS_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for cap build snippets | +| `HTML_VARTABLE_FILE`, `LATEX_VARTABLE_FILE` | Documentation output paths | +| `TYPEDEFS_NEW_METADATA` | Optional: dict enabling DDT member name translation bridge | + +The config file can contain arbitrary Python expressions — computed file lists, +conditional logic, environment-variable lookups — making it very flexible. + +### 2.2 Step-by-step execution pipeline + +``` +1. Import config module dynamically via importlib + +2. gather_variable_definitions() + for each file in VARIABLE_DEFINITION_FILES: + parse_variable_tables(file) [metadata_parser.py] + → metadata_define: OrderedDict[standard_name → [mkcap.Var]] + +3. collect_physics_subroutines() + for each file in SCHEME_FILES: + parse_scheme_tables(file) [metadata_parser.py] + → metadata_request: OrderedDict[standard_name → [mkcap.Var, ...]] + → arguments_request: OrderedDict[scheme → OrderedDict[subroutine → [std_names]]] + → dependencies_request: OrderedDict[scheme → [abs_paths]] + → schemes_in_files: OrderedDict[scheme → abs_path] + +4. compare_metadata() [batch matching] + for each std_name in metadata_request: + check exists in metadata_define + check type/kind/rank compatibility + register unit conversions in var.actions + copy local_name as var.target + → metadata: OrderedDict[std_name → [Var]] (targets and actions set) + +5. check_optional_arguments() [warnings only] + +6. For each requested suite XML: + Suite.parse(xml) [mkstatic.py] → Suite + Group objects + Group.write() → ccpp___cap.F90 + Suite.write() → ccpp__cap.F90 + +7. API.write() [mkstatic.py] + → ccpp_static_api[_].F90 + +8. Write build-system snippets [mkcap.py writers] + → CCPP_CAPS.cmake/mk/sh + → CCPP_SCHEMES.cmake/mk/sh + → CCPP_TYPEDEFS.cmake/mk/sh + → CCPP_API.cmake/sh + +9. mkdoc.metadata_to_html() → HTML variable table + mkdoc.metadata_to_latex() → LaTeX variable table +``` + +### 2.3 Data structures — the "flat dict" model + +Everything in prebuild lives in flat Python `OrderedDict` structures. There is no object +hierarchy; variables are simple Python objects with plain attributes. + +```python +# Top-level data containers +metadata_define: OrderedDict[standard_name → [mkcap.Var]] # 1 Var per std_name +metadata_request: OrderedDict[standard_name → [mkcap.Var, ...]] # N Vars (one per scheme×subroutine) +arguments_request: OrderedDict[scheme_name → OrderedDict[subroutine_name → [std_names]]] +dependencies_request: OrderedDict[scheme_name → [abs_paths]] +schemes_in_files: OrderedDict[scheme_name → abs_path] +``` + +`mkcap.Var` attributes: + +| Attribute | Type | Description | +|---|---|---| +| `standard_name` | str | CF-convention unique identifier | +| `long_name` | str | Human-readable description | +| `units` | str | Physical units | +| `local_name` | str | Fortran local name (may be DDT member reference) | +| `type` | str | Fortran type (real, integer, logical, or DDT name) | +| `kind` | str | Fortran kind parameter | +| `dimensions` | list[str] | Dimension standard names | +| `intent` | str | in / out / inout | +| `active` | str | `'T'`, `'F'`, or expression string | +| `optional` | str | `'T'` or `'F'` | +| `pointer` | bool | Whether Fortran POINTER attribute needed | +| `target` | str | Set during matching: the host model local_name | +| `actions` | dict | `{'in': fn, 'out': fn}` for unit conversions | +| `container` | str | Encoded provenance: `MODULE_foo SCHEME_bar SUBROUTINE_baz` | + +**Performance note on `container` and `target`**: these two attributes act as a lookup +cache computed once during the `compare_metadata()` batch step. The `container` string +encodes where each variable lives in the host model (module and, if applicable, the +DDT member chain). The `target` records the resolved Fortran local name. Both are +computed once and then used directly during Fortran cap generation — no further dictionary +lookups are needed. This is a major contributor to prebuild's speed advantage. + +### 2.4 Metadata parsing and the bridge to capgen + +`metadata_parser.py` is a shared module that acts as a bridge. It detects whether a +metadata section in a Fortran source file uses the old pipe-delimited format (deprecated, +warning emitted) or the new `.meta` format (triggered by `!! \htmlinclude .html` +in the Fortran source comment hook). + +For `.meta` files, `read_new_metadata()` in `metadata_parser.py`: +1. Calls capgen's `metadata_table.parse_metadata_file()` → `[MetadataTable]` +2. Converts each `metavar.Var` to a `mkcap.Var` +3. Normalizes `active` to `'T'`/`'F'`/expression, `optional` to `'T'`/`'F'` + +The `TYPEDEFS_NEW_METADATA` config variable (when provided) triggers an additional +pass via `convert_local_name_from_new_metadata()` which translates flat +standard-name-style local names into DDT member references such as +`Atm(blk_no)%q(:,:,:,graupel_index)`. This is the bridge that makes the newer +`.meta` format work with the older DDT-heavy host model code. + +### 2.5 Variable matching — `compare_metadata()` + +A single batch function processes all matching. For each standard name in `metadata_request`: + +1. Check it exists in `metadata_define` — error if missing +2. Check there is exactly one definition — error if ambiguous +3. Call `var.compatible(other_var)` — checks equality of `standard_name`, `type`, `kind`, and rank +4. Register unit conversions: if units differ, `var.convert_from()` / `var.convert_to()` + stores a conversion function in `var.actions` +5. Check `active` attribute: if host variable is conditionally allocated and scheme variable + is not `optional`, issue a warning (not an error) +6. Copy `local_name` from the define side as `var.target` +7. Build module use list from container strings + +Result: `metadata` dict where each `Var` has `.target` set to the host model local name +and `.actions` populated with any needed unit conversion functions. + +### 2.6 Generated Fortran files + +#### Group cap: `ccpp___cap.F90` + +One module per group. For each CCPP stage (tsinit, init, run, tsfinal, finalize), a subroutine: + +```fortran +module ccpp_suite_A_physics_cap + use scheme_module, only: scheme_run + use host_module_A, only: ddt_A ! DDT, not flat fields + use host_module_B, only: ddt_B + implicit none + contains + + subroutine suite_A_physics_run_cap(ddt_A, ddt_B, im, iaend, ierr, ...) + type(ddt_A_type), intent(inout), target :: ddt_A ! entire DDT passed + type(ddt_B_type), intent(inout), target :: ddt_B + integer, intent(in) :: im, iaend ! loop bounds + integer, intent(out) :: ierr + logical, save :: initialized(200) = .false. + ! optional variable: local pointer, conditionally associated + real(kind_phys), pointer :: opt_var(:) => null() + if (ddt_A%active_flag) then + opt_var => ddt_A%opt_field + end if + ! unit conversion: local variable + real(kind_phys) :: converted_var(im) + converted_var(:) = ddt_B%field(:im) * conversion_factor + ! fixed-index extraction: local pointer for a specific tracer + real(kind_phys), pointer :: q_water_vapor(:,:) => null() + q_water_vapor => ddt_A%q(:,:,ntqv) ! ntqv = water vapor index in tracer array + ! call scheme with loop-bound application and extracted variables at the call site + call scheme_run( & + arg1 = ddt_A%field1(1:im), & ! horizontal loop-bound applied here + arg2 = ddt_A%field2(1:im,:), & ! loop-bound + all levels + qv = q_water_vapor(1:im,:), & ! specific tracer, loop-bound applied + arg3 = converted_var, & ! unit-converted local var + opt_arg = opt_var, & ! optional pointer + ...) + if (ierr /= 0) return + end subroutine +end module +``` + +Key points: +- **DDTs are passed as arguments, not flat fields.** Hundreds of variables arrive as + one or a small number of DDT arguments. This is the fundamental architectural choice + that makes prebuild fast and safe with compiler debugging flags. +- **Two distinct "subsetting" operations happen at the scheme call site:** + 1. *Loop-bound application*: horizontal range `1:im` (or `im` for scalar extents) + applied in the scheme call argument expressions. + 2. *Fixed-index extraction*: a specific element along one dimension is selected, + e.g. `q_water_vapor => ddt%q(:,:,ntqv)` extracts the water vapor tracer from the + full tracer array. A local pointer (or local variable for unit conversions) is + declared just before the scheme call and passed as the scheme argument. The group + cap always receives the full data; these extractions are local to the group cap. +- **Optional variables** are handled by declaring a local `pointer` variable and + conditionally associating it with the DDT field based on the `active` expression. + An unassociated pointer is passed to the scheme if the variable is inactive. This + avoids compiler exceptions when mandatory debugging flags are enabled, because the + unallocated field is never directly referenced — only the already-null pointer is. +- `logical :: initialized(200), save` — per-instance initialization tracking. The + 200 is the maximum number of complete model instances that can coexist in memory + simultaneously (used in ensemble approaches where multiple copies of the full model + state live in memory at once). Each instance has its own initialization flag. +- For the `run` phase, `im` and `iaend` (or similar) carry `horizontal_loop_begin` + and `horizontal_loop_end`, enabling OpenMP thread-level parallelism where each + thread processes a horizontal slice. +- Explicit keyword argument passing in scheme calls. +- Unit conversion: a local variable is declared and populated before the call; the + local variable is then passed to the scheme. +- Error check after each scheme call; returns immediately on error. +- `--debug` flag inserts Fortran array-size assertions. + +#### Suite cap: `ccpp__cap.F90` + +Imports all group cap functions and exposes one function per stage that chains group calls. + +#### Static API: `ccpp_static_api[_].F90` + +A single Fortran module `ccpp_static_api` with one subroutine per stage: + +```fortran +subroutine ccpp_physics_run(cdata, suite_name, group_name, ierr) + character(len=*), intent(in) :: suite_name, group_name + select case(trim(suite_name)) + case('suite_A') + select case(trim(group_name)) + case('physics') + call suite_A_physics_run_cap(cdata, ierr) + ... + end select + ... + end select +end subroutine +``` + +This is the **single entry point** the host model calls. The host model passes `suite_name` +and `group_name` at runtime; the static API dispatches to the appropriate cap function. + +### 2.7 Build system snippet files generated + +Six output files (Makefile, CMakefile, shell source) for three variable sets: + +| File | Content | +|---|---| +| `CCPP_CAPS.cmake` | `set(CAPS /abs/path/cap1.F90 /abs/path/cap2.F90 ...)` | +| `CCPP_SCHEMES.cmake` | `set(SCHEMES /abs/path/scheme1.F90 ...)` | +| `CCPP_TYPEDEFS.cmake` | `set(TYPEDEFS module1 module2 ...)` (module names, not paths) | +| `CCPP_API.cmake` | `set(API /abs/path/ccpp_static_api.F90)` | + +All files are written as `.tmp` first and compared against the existing version; they are +replaced only if the content changed, which avoids unnecessary recompilation of downstream +Fortran targets. + +### 2.8 What `mkcap.py`, `mkstatic.py`, and `mkdoc.py` each do + +**`mkcap.py`**: +- Defines the `mkcap.Var` class (prebuild's variable data class) +- Defines six file-writer classes: `CapsMakefile`, `CapsCMakefile`, `CapsSourcefile`, + `SchemesMakefile`, `SchemesCMakefile`, `SchemesSourcefile`, `TypedefsMakefile`, + `TypedefsCMakefile`, `TypedefsSourcefile` +- Each writer has a `write(file_list)` method that produces a formatted include file +- Does NOT generate any Fortran + +**`mkstatic.py`**: +- Defines `Suite`, `Group`, `Subcycle` classes that parse suite definition XML and + generate Fortran caps +- `Suite.parse()`: reads SDF XML via `xml.etree.ElementTree`, builds `Group` and + `Subcycle` objects +- `Suite.write()`: drives cap generation for all groups and the suite-level cap +- `Group.write()`: generates the group cap Fortran — argument list construction, + module `use` statements, unit conversion code, scheme calls, error handling +- Defines `API` class: generates the static API Fortran module (suite_name/group_name + dispatch switch) +- `CCPP_SUITE_VARIABLES` dict: mandatory variables always included (error message, + error code, loop counter, loop extent) +- Helper functions `extract_parents_and_indices_from_local_name()` and + `extract_dimensions_from_local_name()` handle complex DDT member access like + `Atm(blk_no)%q(:,:,:,graupel_index)` — these are critical for DDT-heavy host models + +**`mkdoc.py`**: +- `metadata_to_html()`: produces an HTML table of all host-model provided variables + (standard_name, long_name, units, rank, type, kind, source, local_name) +- `metadata_to_latex()`: produces a LaTeX table combining host-defined and scheme-requested + variables, annotating which schemes use each variable and whether unit conversion is needed +- Informational outputs only; do not affect the build + +--- + +## 3. ccpp-capgen — detailed analysis + +### 3.1 Command-line arguments + +Entry point: `scripts/ccpp_capgen.py`, `_main_func()`. +Arguments parsed via `framework_env.parse_command_line()` into a `CCPPFrameworkEnv` object: + +| Argument | Required | Purpose | +|---|---|---| +| `--host-files` | yes | `.meta` files or `.txt` indirect file lists | +| `--scheme-files` | yes | Same format | +| `--suites` | yes | `.xml` SDF files or `.txt` lists | +| `--output-root` | no | Directory for generated files | +| `--host-name` | no | If given, generates a host cap | +| `--ccpp-datafile` | no | Path for datatable XML (default: `datatable.xml`) | +| `--kind-type` | no (repeatable) | Fortran kind mappings, syntax `=[:]`. Module defaults to `iso_fortran_env` for ISO_FORTRAN_ENV specs. Examples: `kind_phys=REAL64`, `kind_phys=my_host_kinds:kind_r8`. If omitted, `kind_phys=iso_fortran_env:REAL64` is injected. | +| `--preproc-directives` | no | Fortran preprocessor macros | +| `--use-error-obj` | no | Use error object instead of scalar error variables | +| `--force-overwrite` | no | Always regenerate output | +| `--clean` | no | Remove files listed in datatable and exit | +| `--verbose` | no (repeatable) | Increase log verbosity | + +`CCPPFrameworkEnv` (defined in `framework_env.py`) consolidates all settings into typed +properties and stores a `kind_dict` mapping CCPP kind names to `[kind_spec, module]` pairs. + +### 3.2 Step-by-step execution pipeline + +``` +1. create_file_list() + expand .txt indirect file lists, validate .meta extensions + +2. register_ddts(scheme_files) + pre-scan all scheme .meta files + register DDT type names via register_fortran_ddt_name() + (so the host parser can recognize them as non-intrinsic types) + +3. parse_host_model_files() + for each host .meta file: + metadata_table.parse_metadata_file() → [MetadataTable] + find_associated_fortran_file() → matching .F90 path + parse_fortran_file() → Fortran declarations (via fortran_tools) + check_fortran_against_metadata() → cross-validation (type, kind, rank, intent) + accumulate MetadataSection headers: DDT, module, host types + +4. HostModel(table_dict, host_name, run_env) + process DDT headers: → DDTLibrary + ddt_dict (VarDictionary) + process module/host headers: → main VarDictionary + __var_locations + add ConstituentVarDict synthetically for ccpp_model_constituents_t + +5. API(sdfs, host_model, scheme_headers, run_env) + for each SDF XML: + Suite construction: + auto-create 5 phase groups: register, initialize, timestep_initial, + timestep_final, finalize + parse elements → Group objects (RUN_PHASE_NAME) + parse / tags → Scheme objects in full-phase groups + Suite.analyze(host_model, scheme_library, ddt_library, run_env): + Group.analyze() → Scheme.analyze(): + for each scheme argument: + VarDictionary.find_variable() [scope chain search] + Var.compatible() [→ VarCompatObj with transformations] + loop dim substitution for _run phase + register constituent if constituent=True + variable promotion: group outputs → suite level if needed by later group + +6. ccpp_api.write(outdir, run_env) + suite cap .F90 per suite + group caps (embedded or separate) + host cap .F90 (if --host-name given) + ccpp_kinds.F90 + +7. generate_ccpp_datatable() → datatable.xml +``` + +### 3.3 Object hierarchy + +``` +API (ccpp_suite.py) + └── Suite (extends VarDictionary) [one per SDF XML] + parent → ConstituentVarDict (extends VarDictionary) + parent → API + ├── Group (suite_objects.py, extends VarDictionary) [one per ] + │ call_list: CallList (extends VarDictionary) + │ ├── Subcycle (suite_objects.py) + │ │ └── Scheme (suite_objects.py, extends SuiteObject) + │ └── Scheme (for full-phase groups: init, register, etc.) + └── (auto groups: register, initialize, timestep_initial, + timestep_final, finalize) + +HostModel (host_model.py, extends VarDictionary) + ├── ddt_lib: DDTLibrary + │ └── {ddt_name → MetadataSection} + ├── ddt_dict: VarDictionary (all DDT field variables, expanded) + └── loop_vars: VarDictionary (run-time dimension variables) + +VarDictionary (metavar.py) + ├── {standard_name → Var} + └── parent_dict → VarDictionary ← scope chain for find_variable() + +Var (metavar.py) + └── __prop_dict: {property_name → validated_value} + +VarDDT (ddt_library.py, extends Var) + └── __field: Var | VarDDT ← recursive DDT traversal chain +``` + +### 3.4 Variable matching — scope-chain and VarCompatObj + +Unlike prebuild's single batch `compare_metadata()`, capgen performs incremental, +scope-aware matching during the suite analysis phase. + +For each scheme argument in `Scheme.analyze()`: +1. `VarDictionary.find_variable(standard_name)` — searches scope chain: + local group dict → suite dict → ConstituentVarDict → host model dict +2. `Var.compatible(other, run_env)` returns a `VarCompatObj` — not a bool. + `VarCompatObj` carries: + - Whether the variables are equivalent (no transformation needed) + - Whether they are compatible with transformations (unit conversion, dimension + substitution, `top_at_one` flip) + - The reason for any incompatibility (for error messages) +3. For `_run` phase: `horizontal_dimension` is automatically substituted with + `horizontal_loop_begin:horizontal_loop_end` +4. For `constituent = True` variables: auto-registered in `ConstituentVarDict`; + allocation/management code is generated +5. Variable promotion: if a Group produces a variable needed by a later Group, it is + promoted to Suite-level scope + +`VarCompatObj` compatibility considers: +- Type equality +- Kind equality (with ISO kind aliases) +- Units compatibility (triggers unit conversion if compatible) +- Dimension substitutability (horizontal loop vs. full dimension, vertical extent) +- `top_at_one` orientation (triggers flip if needed) +- `protected` status (cannot be an output if protected) +- `CCPP_HORIZONTAL_DIMENSIONS`, `CCPP_VERTICAL_DIMENSIONS`, `CCPP_LOOP_DIM_SUBSTS` + from `var_props.py` + +### 3.5 `metavar.Var` properties + +`metavar.Var` stores all properties in a validated `__prop_dict`. Properties: + +**Specification properties** (all metadata contexts): + +| Property | Type | Notes | +|---|---|---| +| `local_name` | str | Valid Fortran identifier | +| `standard_name` | str | CF-convention, lowercase+underscores | +| `long_name` | str | Human-readable description | +| `units` | str | Physical units string | +| `dimensions` | list | Dimension standard names or `()` | +| `type` | str | Intrinsic or registered DDT name | +| `kind` | str | Fortran kind parameter | +| `active` | str | Conditional allocation expression | +| `optional` | bool | Whether scheme can handle missing var | +| `protected` | bool | Cannot be written by schemes | +| `allocatable` | bool | Has ALLOCATABLE attribute | +| `state_variable` | bool | Persists across timesteps | +| `persistence` | str | `timestep` or `run` | +| `default_value` | str | Fortran expression | +| `diagnostic_name` | str | Diagnostic output name | +| `target` | bool | Has TARGET attribute | +| `polymorphic` | bool | CLASS(*) type | +| `top_at_one` | bool | Vertical ordering: top at index 1 | + +**Scheme-only properties**: + +| Property | Type | Notes | +|---|---|---| +| `intent` | str | in / out / inout | + +**Constituent properties**: + +| Property | Type | Notes | +|---|---|---| +| `constituent` | bool | Is a CCPP-managed constituent (tracer) | +| `advected` | bool | Is advected by the dynamical core | +| `molar_mass` | float | Molecular weight (positive) | + +### 3.6 Capgen-only features + +**Fortran cross-validation** (`check_fortran_against_metadata()`): +- Parses the actual `.F90` file alongside the `.meta` file +- Checks that every metadata entry matches the real Fortran declaration: + variable count, local_name, type, kind, intent (for schemes), dimension rank/names +- Catches bugs where metadata was updated but the Fortran source was not (or vice versa) + +**State machine** (`ccpp_state_machine.py`, `state_machine.py`): +- `CCPP_STATE_MACH`: a `StateMachine` instance with 6 transitions +- Valid state sequence: `register → uninitialized → initialized → in_time_step` +- Suite caps include a `character(len=16) :: ccpp_suite_state` variable +- State-checking code at the start of each phase function enforces correct call ordering +- `CCPP_STATE_MACH.function_match()` uses compiled regex to identify which CCPP phase + a subroutine name belongs to + +**Constituent variable support** (`constituents.py`): +- `ConstituentVarDict` (extends `VarDictionary`) manages traceable species (tracers) +- When a scheme declares `constituent = True`, `find_variable()` auto-creates the variable +- Allocation code for the constituent array is auto-generated +- Constants: `CONST_DDT_NAME = "ccpp_model_constituents_t"`, + `CONST_PROP_TYPE = "ccpp_constituent_properties_t"` + +**DDT library** (`ddt_library.py`): +- `VarDDT(Var)`: represents a DDT field variable at any nesting level +- Traversal chain: `VarDDT → VarDDT → ... → Var` (innermost is the actual leaf field) +- `DDTLibrary`: dictionary of DDT `MetadataSection` objects +- `collect_ddt_fields()` expands DDT variables into component fields in `ddt_dict` + +**Host cap generation** (`host_cap.py`): +- Generated only when `--host-name` is given +- Produces `_ccpp_cap.F90` +- Subroutines: `_ccpp_physics_(api_vars)` + that call into suite cap functions + +**`ccpp_kinds.F90`**: +- Simple Fortran module `ccpp_kinds`. **Always generated**, even when no `--kind-type` + is supplied (in that case `kind_phys=iso_fortran_env:REAL64` is injected + automatically and an INFO log line is emitted). +- One `use , only: ` line per module (modules sorted; specs deduped per + module). Each kind is then re-exported as + `integer, parameter, public :: = `. +- Supports host-supplied kind modules: `--kind-type kind_phys=my_host_kinds:kind_r8` + emits `use my_host_kinds, only: kind_r8` and + `integer, parameter, public :: kind_phys = kind_r8`. +- Listed in `` of `datatable.xml` (matches original capgen) so + the build system picks it up via `ccpp_datafile.py --ccpp-files`. +- USEd by all generated Fortran files that declare kind-typed variables — the group + cap, the suite types module, and the suite data module. The static API and suite + cap have no kind references and do not USE it. + +**Datatable XML** (`ccpp_datafile.py`): +- Produced after generation; lists all generated files, scheme entries, variable properties, + suite configurations +- Queryable by the build system via `ccpp_datafile.py --suite-files` etc. +- Supports `--clean` workflow: reads the file list, removes all generated files, deletes itself +- `DatatableReport` class provides a programmatic query API + +**In-memory database** (`ccpp_database_obj.py`): +- `CCPPDatabaseObj`: wraps `HostModel` and `API` for programmatic access to capgen results +- Returned when `capgen()` is called with `return_db=True` +- Provides `host_model_dict()`, `suite_list()`, `constituent_dictionary(suite)` + +**Variable tracking tool** (`ccpp_track_variables.py`): +- Standalone diagnostic: traces a specific variable through a suite, showing which schemes + use it and with what intent +- Uses prebuild's `import_config` and capgen's `Suite`/`parse_metadata_file` together + +**Fortran-to-metadata tool** (`ccpp_fortran_to_metadata.py`): +- Standalone utility: parses annotated Fortran source files and generates skeleton `.meta` + files — used to bootstrap new scheme metadata + +--- + +## 4. Shared infrastructure + +### 4.1 Module sharing map + +| Module | Used by prebuild | Used by capgen | Notes | +|---|---|---|---| +| `metadata_parser.py` | yes | partial | **Bridge module**: calls capgen's parser, returns mkcap.Var | +| `metadata_table.py` | via bridge | yes (primary) | Native `.meta` format parser | +| `metavar.py` | no | yes | Primary `Var` class, `VarDictionary` | +| `var_props.py` | no | yes | `VariableProperty`, `VarCompatObj`, dimension constants | +| `mkcap.py` | yes | no | `mkcap.Var` class + build-snippet writers | +| `mkstatic.py` | yes | no | Suite/Group/API Fortran generators | +| `mkdoc.py` | yes | no | HTML/LaTeX documentation generators | +| `common.py` | yes | partial | `CCPP_STAGES`, container encoding | +| `framework_env.py` | dummy instance | yes (primary) | `CCPPFrameworkEnv` | +| `file_utils.py` | no | yes | `create_file_list`, `move_modified_files` | +| `code_block.py` | no | yes | Structured Fortran output | +| `ddt_library.py` | no | yes | `DDTLibrary`, `VarDDT` | +| `host_model.py` | no | yes | `HostModel` class | +| `host_cap.py` | no | yes | Host cap generation | +| `ccpp_suite.py` | no | yes | `Suite`, `API` classes | +| `suite_objects.py` | no | yes | `Scheme`, `Group`, `Subcycle`, `CallList` | +| `constituents.py` | no | yes | `ConstituentVarDict` | +| `ccpp_datafile.py` | no | yes | Datatable XML | +| `ccpp_database_obj.py` | no | yes | `CCPPDatabaseObj` | +| `ccpp_state_machine.py` | no | yes | `CCPP_STATE_MACH` | +| `state_machine.py` | no | yes | `StateMachine` base class | +| `ccpp_fortran_to_metadata.py` | no | yes | Fortran→metadata bootstrap tool | +| `ccpp_track_variables.py` | partial | partial | Uses both worlds | + +**The key architectural debt**: `metadata_parser.py` is a prebuild module that internally +calls capgen's `metadata_table.parse_metadata_file()` and converts results to `mkcap.Var` +objects. This creates a one-way dependency (prebuild → capgen's parser infrastructure) +while presenting a prebuild-style API to `ccpp_prebuild.py`. It exists only because +prebuild predates the `.meta` format. + +### 4.2 The `.meta` file format + +The `.meta` format is the native format for capgen and the expected format for all new +scheme development. The Fortran source file contains a comment hook pointing to the `.meta` +file: + +```fortran +!! \section arg_table_scheme_name_run Argument Table +!! \htmlinclude scheme_name_run.html +``` + +The `.meta` file itself uses an INI-style format: + +```ini +[ccpp-table-properties] + name = scheme_name + type = scheme + source_path = ../src + dependencies_path = ../some/path + dependencies = utility_module.F90, another.F90 + +[ccpp-arg-table] + name = scheme_name_run + type = scheme +[ im ] + standard_name = horizontal_dimension + long_name = horizontal dimension + units = count + type = integer + dimensions = () + intent = in +[ dz ] + standard_name = layer_thickness + long_name = thickness of each model layer + units = m + type = real + kind = kind_phys + dimensions = (horizontal_dimension, vertical_layer_dimension) + intent = in +``` + +Multiple `[ccpp-arg-table]` sections are allowed in a scheme file (one per phase: +`_init`, `_run`, `_finalize`, `_timestep_init`, `_timestep_finalize`). +Singleton tables (DDT, module, host) allow only one section. + +The three table-level properties in `[ccpp-table-properties]` that carry path information: + +| Property | Purpose | Resolution | +|---|---|---| +| `source_path` | Relative path from the `.meta` file's directory to the directory containing the corresponding `.F90` Fortran source file | `os.path.normpath(os.path.join(meta_dir, source_path))`. Defaults to `meta_dir` when absent. | +| `dependencies_path` | Optional subdirectory relative to `meta_dir`; used as the base directory for resolving entries in `dependencies` | `os.path.normpath(os.path.join(meta_dir, dependencies_path))`. Defaults to `meta_dir` when absent. | +| `dependencies` | Comma-separated list of dependency file names or relative paths | Each entry resolved via `os.path.normpath(os.path.join(dep_base, entry))` where `dep_base` is the resolved `dependencies_path`. The value `none` is ignored. | + +**Implementation note — `flush_table_props` pattern:** The INI parser processes the +`[ccpp-table-properties]` and `[ccpp-arg-table]` headers in one streaming pass. Extra +table-level keys (`source_path`, `dependencies_path`, `dependencies`) are collected in a +`pending_props` dict alongside `name` and `type`. The parser must apply these properties +to the `MetadataTable` object — via a `flush_table_props()` call — at every +state-transition point (first `[ccpp-arg-table]` header, next `[ccpp-table-properties]` +header, and end-of-file) **before** resetting `pending_props`. Without this, the extra +properties are silently discarded. + +### 4.3 Variable property validation (`var_props.py`) + +`VariableProperty` encapsulates a single metadata property with its name, Python type, +optionality, default, valid-value constraints, and a check function. Check functions used: + +| Checker | What it validates | +|---|---| +| `check_local_name` | Valid Fortran identifier | +| `check_cf_standard_name` | Lowercase, underscores, alphanumeric only | +| `check_fortran_type` | Intrinsic type or registered DDT name | +| `check_units` | Valid unit string (normalizes `+` in exponents) | +| `check_dimensions` | Valid dimension specification | +| `check_default_value` | Valid Fortran expression | +| `check_molar_mass` | Positive float (for constituents) | + +`CCPP_HORIZONTAL_DIMENSIONS`, `CCPP_VERTICAL_DIMENSIONS`, `CCPP_LOOP_DIM_SUBSTS` +in `var_props.py` define the recognized dimension forms and the run-time substitution +map (e.g., `horizontal_dimension → horizontal_loop_begin:horizontal_loop_end`). + +--- + +## 5. Feature comparison + +| Feature | prebuild | capgen | Notes | +|---|---|---|---| +| **Input formats** | | | | +| Native `.meta` format | via bridge | yes | | +| Old pipe-delimited format | deprecated warn | not supported | | +| **Parsing and validation** | | | | +| Fortran source cross-validation | no | yes | capgen parses actual .F90 to cross-check | +| Preprocessor directive support | no | yes | `--preproc-directives` | +| **Variable handling** | | | | +| Variable data class | `mkcap.Var` (flat attrs) | `metavar.Var` (validated prop dict) | | +| Scope-chain variable search | no | yes | group→suite→constituent→host | +| Variable promotion group→suite | no | yes | | +| Unit conversion | yes | yes | | +| Optional/active variables | yes (fully) | yes | both: local pointer + conditional ASSOCIATE | +| DDT library (first-class) | no | yes | `VarDDT` recursive chain | +| **Suite and cap generation** | | | | +| Suite definition (SDF XML) | yes | yes | Same XML format | +| Subcycle loops | yes | yes | | +| State machine in generated caps | no | yes | Runtime state enforcement | +| Static API module (dispatch switch) | yes | no | `ccpp_static_api.F90` | +| Host cap generation | no | yes | `_ccpp_cap.F90` | +| `ccpp_kinds.F90` | no | yes | | +| **Constituent/tracer support** | | | | +| Constituent variable management | no | yes | Auto-allocation, `ConstituentVarDict` | +| **Build system output** | | | | +| CMake/Makefile file-list snippets | yes | no | Six snippet files | +| Datatable XML (queryable) | no | yes | `ccpp_datafile.py` | +| Clean via datatable | no | yes | | +| **Documentation** | | | | +| HTML variable table | yes | no (stub, raises error) | `mkdoc.metadata_to_html` | +| LaTeX variable table | yes | no | `mkdoc.metadata_to_latex` | +| **Developer tools** | | | | +| Variable tracking diagnostic | yes | no | `ccpp_track_variables.py` | +| Fortran-to-metadata bootstrap | no | yes | `ccpp_fortran_to_metadata.py` | +| **Runtime API** | | | | +| In-memory database object | no | yes | `CCPPDatabaseObj` | +| **Debug / developer aids** | | | | +| Debug array-size checks in caps | yes (`--debug`) | no | | +| Namespace suffix for API name | yes (`--namespace`) | no | | +| **Configuration** | | | | +| Config mechanism | Python module (flexible) | CLI args only | | + +**Known gaps and corrections:** + +- Capgen's `--generate-docfiles` is declared in the CLI but raises + `CCPPError("not yet supported")` — documentation generation is unimplemented. +- Prebuild handles `TYPEDEFS_NEW_METADATA` for mixed old/new metadata deployments; + capgen has no equivalent because it only accepts the new format. +- Capgen validates Fortran source against metadata; prebuild trusts metadata and never + reads Fortran code. +- Capgen has no `--namespace` equivalent for the generated API module name. +- Capgen's `CCPPDatabaseObj` and datatable XML allow programmatic querying; prebuild + has no equivalent. +- Prebuild's static API pattern (single Fortran module with runtime dispatch) is absent + from capgen, which uses a different host-cap integration model. +- **Capgen cannot pass DDTs to group caps** — it passes everything as flat fields. + Despite considerable effort by multiple developers, this has not been fixed. This is + the primary reason capgen is being abandoned. +- **Capgen does not support multiple model instances in memory** (ensemble approach). + Prebuild's `initialized(200)` array handles this correctly. +- **Capgen does not own or allocate any data.** Wait — this is a prebuild characteristic. + Capgen *does* allocate data for physics-internal variables (variables used only within + the physics, not provided by the host model) at the suite level. Prebuild requires the + host model to provide and own all data, including any physics-internal scratch space. + +--- + +## 6. Build system integration + +### 6.1 How a host model invokes ccpp-prebuild + +Direct call (as in the test suite): +```bash +python ../../scripts/ccpp_prebuild.py \ + --config=ccpp_prebuild_config.py \ + --builddir=build \ + --suites=suite_A,suite_B \ + [--debug] [--namespace mymodel] +``` + +Typical CMake integration: +```cmake +# Run prebuild at configure time +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_prebuild.py + --config=${HOST_CCPP_PREBUILD_CONFIG} + --builddir=${CMAKE_CURRENT_BINARY_DIR} + --suites=${CCPP_SUITES} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + RESULT_VARIABLE PREBUILD_RESULT +) +if(NOT PREBUILD_RESULT EQUAL 0) + message(FATAL_ERROR "ccpp_prebuild.py failed") +endif() + +# Consume the generated snippet files +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_CAPS.cmake) # → ${CAPS} +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_SCHEMES.cmake) # → ${SCHEMES} +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_TYPEDEFS.cmake) # → ${TYPEDEFS} +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_API.cmake) # → ${API} + +add_library(ccpp_physics OBJECT ${CAPS} ${SCHEMES} ${API}) +``` + +### 6.2 How a host model invokes ccpp-capgen + +Direct call: +```bash +python scripts/ccpp_capgen.py \ + --host-files host_data.meta,host_model.meta \ + --scheme-files scheme1.meta,scheme2.meta \ + --suites suite_A.xml,suite_B.xml \ + --output-root ${BUILD_DIR}/ccpp \ + --host-name my_host \ + --kind-type kind_phys=REAL64 \ + --ccpp-datafile ${BUILD_DIR}/ccpp/datatable.xml +``` + +Typical CMake integration: +```cmake +# Run capgen at configure time +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_capgen.py + --host-files ${HOST_META_FILES} + --scheme-files ${SCHEME_META_FILES} + --suites ${SUITE_SDFS} + --output-root ${CMAKE_CURRENT_BINARY_DIR}/ccpp + --host-name ${HOST_MODEL_NAME} + --ccpp-datafile ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml + RESULT_VARIABLE CAPGEN_RESULT +) + +# Query the datatable for generated file lists +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_datafile.py + ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml + --suite-files + OUTPUT_VARIABLE SUITE_CAPS OUTPUT_STRIP_TRAILING_WHITESPACE +) +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_datafile.py + ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml + --host-files + OUTPUT_VARIABLE HOST_CAP OUTPUT_STRIP_TRAILING_WHITESPACE +) + +add_library(ccpp_physics OBJECT ${SUITE_CAPS} ${HOST_CAP}) +``` + +### 6.3 Available datatable query flags + +``` +--host-files → generated host cap .F90 files +--suite-files → generated suite cap .F90 files +--utility-files → generated utility .F90 files (e.g. ccpp_kinds.F90) +--ccpp-files → all generated .F90 files +--process-list → physics process types in the suite +--module-list → Fortran module names needed +--dependencies → scheme dependency files +--suite-list → configured suite names +--required-variables → variables required by all suites +--input-variables → input-only variables for a suite +--output-variables → output variables for a suite +--host-variables → variables provided by the host model +``` + +--- + +## 7. Key architectural differences + +### 7.1 Data model + +| Dimension | ccpp-prebuild | ccpp-capgen | +|---|---|---| +| Variable representation | `mkcap.Var` with plain Python attributes | `metavar.Var` with validated `__prop_dict` | +| Variable storage | Two flat `OrderedDict`s | Scope-chain `VarDictionary` tree | +| Container encoding | Encoded string: `MODULE_foo SCHEME_bar SUBROUTINE_baz` | Explicit class hierarchy | +| DDT handling | Encoded as string in `local_name`; helper regexes to extract | First-class `VarDDT` recursive chain | +| Variable matching | One batch `compare_metadata()` call | Incremental during suite analysis | +| Matching result | `bool` + side effects on `.target` / `.actions` | Rich `VarCompatObj` with transformation info | +| **Cap argument style** | **DDTs passed to group caps** | **Flat fields passed to group caps** | +| Subsetting location | At the scheme call site inside the group cap | Done at a higher level, before group cap | +| Data ownership | Host model owns all data including physics-internal | Capgen allocates physics-internal suite-level data | +| Multiple model instances | Yes — `initialized(200)` array, one flag per instance | No | +| Optional variable handling | Local pointer, conditionally associated | Same mechanism, but blocked by flat-field issue | + +### 7.2 Error handling + +| Aspect | ccpp-prebuild | ccpp-capgen | +|---|---|---| +| Style | `(success, result)` tuples + `logging.error()` | `CCPPError` / `ParseInternalError` exceptions | +| Collection | Errors accumulate via `logging`; `main()` checks success | Raised immediately at point of detection | +| Location info | Filename from context; line numbers sometimes | `ParseContext` objects with file + line number | +| User errors vs bugs | Not distinguished | `CCPPError` (user) vs `ParseInternalError` (programmer) | + +### 7.3 Extensibility + +| Aspect | ccpp-prebuild | ccpp-capgen | +|---|---|---| +| New metadata property | Add to `VALID_ITEMS` dict + `mkcap.Var` attribute | Add one `VariableProperty` entry + checker fn | +| New CCPP phase | Update `CCPP_STAGES` + regenerate static API template | Add one transition tuple to `CCPP_STATE_MACH` | +| New compatibility rule | Modify `var.compatible()` in `mkcap.py` | Extend `VarCompatObj` in `var_props.py` | +| New host model | Write a new Python config file | New `.meta` files + CLI invocation | + +### 7.4 Performance + +Prebuild generates caps for multiple suites in seconds. Capgen, on the same suite set +with the same physics, takes more than 10 minutes. Two independent causes: + +**Cause 1 — Repeated scope-chain traversal.** Every variable lookup in capgen traverses +a five-level `VarDictionary` parent chain (group → suite → constituent dict → host model +→ DDT dict) for every scheme argument in every group in every suite. Prebuild's +`compare_metadata()` does one flat dict lookup per standard name, once, and caches the +result in `var.container` and `var.target`. All subsequent use during Fortran generation +reads these cached attributes directly. + +**Cause 2 — Flat-field cap arguments.** This is likely the dominant cost. Capgen resolves +every scheme argument down to its individual flat field, generates a `use` statement and +an explicit argument for each one, and emits them in the generated Fortran. A DDT with +200 fields becomes 200 individual argument declarations, 200 `use` statements, and 200 +argument positions in the scheme call. Prebuild passes the DDT itself — one argument, +one `use` statement — and then subsets at the call site. + +**Consequence for correctness.** Passing flat fields in capgen also breaks optional +variable handling under Fortran compiler debugging flags. When a field inside a DDT is +conditionally allocated (optional), passing it as a flat field requires dereferencing the +DDT to extract the field — which the compiler will flag as an error if debugging is on +and the field happens to be unallocated. Prebuild avoids this entirely by passing the +DDT and using a local pointer at the scheme call site. + +### 7.5 Team comprehension and maintainability + +This is the critical real-world difference. `ccpp-prebuild` is understood by the whole +team because it is procedural Python: you can read `ccpp_prebuild.py` top-to-bottom and +follow what happens. The data structures are flat dicts; the control flow is linear. + +`ccpp-capgen` has a five-level class hierarchy, scope-chain dictionary lookups, +`VarCompatObj` carrying transformation state, `ConstituentVarDict` as a pluggable +scope-chain node, and a `StateMachine` with regex-based dispatch. No remaining team +member fully understands all of it. Development is extremely slow and risky. + +The failed effort to make capgen pass DDTs instead of flat fields is the concrete proof +point: three developers spent considerable time and could not fix it without fully +understanding the interplay between `VarDDT`, `DDTLibrary`, `VarDictionary` scope chains, +and the Fortran writer. This is the proximate reason for the redesign. + +--- + +## 8. Design considerations for the redesign + +The following observations from this analysis should inform the redesign: + +### 8.1 What to keep from prebuild +- Procedural, top-down control flow — easy to read and debug +- Config file as a Python module — extremely flexible without adding CLI arguments +- The static API pattern (`ccpp_static_api.F90` with runtime suite/group dispatch) — + proven, simple integration for the host model +- **DDT arguments in group caps** — pass DDTs, not flat fields; this is the core correctness + and performance requirement +- **Subsetting at the scheme call site** — group caps always receive full data; loop-bound + application and fixed-index extraction happen in the individual scheme call expressions + or via a local variable/pointer declared just before the call +- **Optional variable pattern** — local pointer declared in the group cap, conditionally + associated based on the `active` expression, then passed to the scheme; this is safe + under all compiler debugging modes +- The `initialized(N)` per-instance tracking — handles multiple simultaneous model + instances in memory (ensemble approach); `N` is the max number of instances +- **Framework-owned data needs a simpler design** — capgen's variable promotion and + `ConstituentVarDict` scope-chain approach is too complex; a cleaner mechanism for + framework-allocated physics-internal data is needed (to be designed) +- HTML and LaTeX documentation generation +- The six CMake/Makefile/shell snippet output files — simple and direct (can be revisited) + +### 8.2 What to keep from capgen +- Native `.meta` file parsing (eliminate the `metadata_parser.py` bridge entirely) +- Fortran source cross-validation (`check_fortran_against_metadata()`) — catches real bugs +- Rich compatibility reporting (`VarCompatObj`-style) — better error messages +- `ccpp_kinds.F90` generation — important for portability +- Datatable XML as output accounting (strictly better than six include files) +- `--preproc-directives` support +- Constituent variable support (needed for CAM-SIMA) +- State machine enforcement (optional feature, but architecturally clean) + +### 8.3 What to eliminate +- The `mkcap.Var` / `metavar.Var` duality — one variable class, natively reading `.meta` +- The `metadata_parser.py` bridge module — it exists only because of the old format +- The scope-chain `VarDictionary` hierarchy — replace with flat, explicit lookup: + one host dict, one scheme dict; no parent-chain traversal +- The five-level class inheritance (Suite → VarDictionary → ParseSource → ...) +- `ConstituentVarDict` as a scope-chain node — a simple explicit constituent registry suffices +- Capgen's variable promotion (group → suite level) — this complexity exists only because + capgen allocates physics-internal data; if the host always owns all data, promotion + is unnecessary +- Capgen's flat-field cap generation — DDT arguments must be the foundation + +### 8.4 Framework-owned data — open design question + +Capgen's variable promotion mechanism (promoting a variable from group scope to suite scope +when a later group needs it) and the `ConstituentVarDict` complexity exist because capgen +allocates and manages physics-internal data — variables used only within the physics, +not visible to the host model. This capability is **wanted** in the redesign: the host +model should not have to declare and own scratch variables that are purely internal to the +physics. + +The problem is not the concept but the implementation. Capgen's approach — weaving +framework-allocated variables into the `VarDictionary` scope chain and promoting them +upward — produces the complexity that made capgen unmaintainable. + +**Open question for the redesign:** What is a simpler mechanism for the framework to +allocate, own, and pass physics-internal variables? Candidate approaches (to be evaluated +with real-world examples): + +- A completely separate, flat "framework data" dictionary, distinct from the host variable + lookup, populated during analysis and passed explicitly to the caps as a dedicated + argument (e.g., a framework-managed DDT or allocatable array container). +- A simplified promotion concept: variables are statically promoted to the widest scope + that needs them during the analysis phase, but stored in a simple flat dict rather than + via a scope-chain lookup. +- Constituent variables (tracers) as a special sub-case with their own well-defined + allocation interface, separate from generic physics-internal data. + +This question will be revisited once real-world examples clarify how many and what kind of +physics-internal variables actually need to be managed. + +### 8.5 Critical design decisions for the redesign prompt + +1. **DDT cap arguments are non-negotiable.** Group caps must receive DDTs. The entire + subsetting, optional-variable, and performance story depends on this. + +2. **Data ownership**: host-owns-all (prebuild model) vs. generator-allocates-internals + (capgen model). This single decision determines whether variable promotion and + suite-level allocation are needed. + +3. **Integration pattern**: static API (prebuild style, `suite_name` + `group_name` dispatch) + vs. host cap (capgen style, separate host-side Fortran glue). Models currently using + each pattern depend on it. + +4. **Config mechanism**: Python module (prebuild style, flexible) vs. pure CLI + file lists + (capgen style, scriptable). The Python module config is very powerful for complex models. + +5. **DDT member access parsing**: `extract_parents_and_indices_from_local_name()` and + `extract_dimensions_from_local_name()` in `mkstatic.py` handle expressions like + `Atm(blk_no)%q(:,:,:,graupel_index)`. The redesign needs a clean, explicit design for + parsing and emitting these — not an afterthought regex patch. + +6. **Output accounting**: datatable XML (capgen) is the right answer. The six CMake snippet + files (prebuild) are redundant and harder to extend. + +7. **Multiple model instances**: the redesign must preserve the `initialized(N)` pattern + or an equivalent. The value of `N` may need to be configurable. + +8. **Backward compatibility of generated Fortran interfaces**: real-world model examples + will define exactly which naming conventions, argument orders, and module structures the + host models depend on. + +### 8.6 Implementation decisions made during redesign + +The following decisions were made during implementation of `capgen` and are recorded +here as amendments to the analysis above. + +**State machine parameters are local to each generated group cap module.** +The original redesign prompt described the integer state constants as coming from a +shared framework library module. In practice they are generated as `private` named +parameters directly inside each group cap module: + +```fortran +integer, parameter, private :: CCPP_GROUP_UNINITIALIZED = 0 +integer, parameter, private :: CCPP_GROUP_INITIALIZED = 1 +integer, parameter, private :: CCPP_GROUP_IN_TIMESTEP = 2 +``` + +This keeps generated files self-contained — no implicit dependency on a framework +runtime library at the caps level. The values are replicated across all generated group +cap files, but the names are the contract. + +**`source_path` is used by the validator, not the generator.** +The generator trusts metadata and never opens Fortran source files. `source_path` is +meaningful only to the standalone validator tool, which uses it to auto-discover the +`.F90` file paired with each `.meta` file (same base name, different directory). + +**`dependencies` paths are written to `datatable.xml`.** +The resolved absolute paths from each scheme's `dependencies` table-level property are +collected and written to the `` section of `datatable.xml`, sorted and +deduplicated. The CMake build system reads these via `ccpp_datafile.py` to add external +dependency files to the build graph. + +**Optional variable (pointer wrapper) implementation decisions.** +Optional arguments (Case 2 and Case 4) use per-suite Fortran derived types for pointer +wrappers. All unique `(type, kind, rank)` combinations needed by any optional arg across +all groups in a suite are collected and written to `ccpp__types.F90`. Each type +name is generated as `{type}_{kind}_rank{N}_ptr_type` (e.g. `real_kind_phys_rank1_ptr_type`). +Group cap modules `USE` this file. The types file is omitted entirely when no optional +args exist in the suite. The active condition for a pointer assignment is inherited from +the **host variable's** `active` attribute when the scheme itself specifies no `active`. + +**Character length (`len=N` / `len=*`) rules.** +Character kind declarations follow specific compatibility rules enforced by the resolver: + +- `len=*` is valid only where a character variable is **passed**, never where its + storage is **defined**. Host and DDT metadata must give every character variable + a concrete `len=N`; so must the first `intent=out` scheme that defines a + suite-owned character variable (it freezes the storage the framework allocates in + `ccpp__data`). `len=*` in any of those positions is a **metadata error**. + Control variables are exempt — they are pass-through dummy arguments the caps + declare `character(len=*)`. +- `len=*` in a **consuming/later** scheme is always compatible with the defining + `len=` — assumed-length dummy arguments accept any declared length. No transform. +- Matching specific `len=N` on both sides requires no transform (naturally equal). +- Mismatched specific lengths (`len=512` definer vs `len=128` consumer) are a + **metadata error**; the consuming scheme must declare `len=*` or match exactly. + +The resolver raises `CCPPError` for the illegal cases. No kind transform is ever generated +for character variables — lengths are a Fortran compatibility constraint, not a unit conversion. + +**`source_path` is used by the validator, not the generator.** +The group cap's `state_alloc` subroutine always takes `number_of_instances` as an +explicit `intent(in)` integer argument — it never USEs any host module to obtain it. +The call chain is: `ccpp_init` → `_init` → each group's `state_alloc`. At each +level the argument is conditional: + +- **Multi-instance host** (`number_of_instances` declared in host metadata with local + name e.g. `ninstances`): + - `ccpp_init(suite_name, ninstances, errmsg, errflg)` — static API receives it + - `_init(ninstances, errmsg, errflg)` — suite cap threads it through + - `state_alloc(ninstances, errmsg, errflg)` — group cap allocates array of that size +- **Single-instance host** (no `number_of_instances` in host metadata): + - All three signatures omit the argument + - `state_alloc(1, errmsg, errflg)` — the literal `1` is passed + +State array **indexing** uses `instance_number`'s local name (e.g. `inst_num`) from +the control metadata. For single-instance hosts the literal `1` is used. `instance_number` +is injected into the group cap's `_init` and `_final` subroutine signatures even when no +scheme in those phases uses it — the state guard and state transition require it: + +```fortran +subroutine ccpp___init(inst_num, ...) + if (ccpp_group_state(inst_num) >= CCPP_GROUP_INITIALIZED) return + ! ... scheme _init calls ... + ccpp_group_state(inst_num) = CCPP_GROUP_INITIALIZED +``` + +This injection does **not** happen for `_run`, `_timestep_init`, or `_timestep_final` +unless a scheme in those phases explicitly requests `instance_number`. The suite cap's +`_physics_init` and `_physics_final` dispatch subroutines similarly pass +`instance_number` to the group cap calls when the host provides it. + +**Control variable validation — flat unconditional required set.** +All required control variables (`suite_name`, `horizontal_loop_begin`, `horizontal_loop_end`, +`thread_number`, `number_of_threads`, `number_of_physics_threads`, `ccpp_error_code`, +`ccpp_error_message`, `instance_number`) are unconditional — every host must declare all +of them. Single-threaded or single-instance models pass `1` or `''` for any they don't +actively use. The generator validates the complete set after parsing host metadata, +collects all missing-variable errors together, and halts before emitting any code. +`instance_number` in particular is NOT conditional on `instance_dimension` usage — +it is always required. + +**`group_name` is conditionally included, not in the required set.** +`group_name` is included in the static API signature only if the host declares it in +their `type=control` table. When absent, the cap calls all groups unconditionally. The +generator warns (not errors) if `group_name` is absent and any suite has multiple groups. +When present: a required (non-optional) character argument; `''` or `'all'` calls all +groups in order; any other value dispatches to the named group only. + +**`horizontal_loop_extent` eliminated; schemes always use `horizontal_dimension`.** +Scheme metadata always declares `horizontal_dimension` as the horizontal extent +dimension, regardless of phase. There is no `horizontal_loop_extent` standard name in +the new design. The distinction between run-phase chunked processing and full-domain +init/final processing is handled entirely at the host level — the host passes actual +chunk bounds to `ccpp_physics_run` and `1`/`ncols` to all other phases. The cap always +generates `(horizontal_loop_begin:horizontal_loop_end)` for scheme call-site array +slices. For suite-owned array allocation sizing, `horizontal_dimension` from the host +`type=host` table (module USE) is used directly. This separation means allocation +correctness does not depend on the host passing any particular control variable values. + +**Uniform signature across all `ccpp_physics_*` entry points.** +All five physics entry points (`ccpp_physics_init`, `ccpp_physics_timestep_init`, +`ccpp_physics_run`, `ccpp_physics_timestep_final`, `ccpp_physics_final`) share the +same control argument set. No per-phase signature variations. `horizontal_loop_begin` +and `horizontal_loop_end` are in scope for all phases — a `_init` scheme that declares +`horizontal_dimension` correctly receives `(lb:ub)` slicing just as a `_run` scheme +would, with the host responsible for passing the right values. + +**Both `ccpp_physics_final` and `ccpp_final` are silently idempotent.** +Symmetric to `ccpp_physics_init`'s silent skip when already `INITIALIZED`, +both final-path entry points return cleanly with `errflg=0` on every repeat +invocation. Three cap levels participate: + +- The suite-cap `_physics_final` dispatcher silent-returns when + `ccpp_suite_state` is unallocated (last-instance post-`ccpp_final` deallocation) + or `ccpp_suite_state(inst_num) == CCPP_SUITE_UNREGISTERED` (any other + instance post-`ccpp_final`). The `state /= FRAMEWORK_INITIALIZED` error is + retained so calling `physics_final` after only `ccpp_register` (no `ccpp_init`) + still errors. +- The group-cap `_final` entry guard silent-returns when + `ccpp_group_state(inst_num) == CCPP_GROUP_UNINITIALIZED`. (Since `UNINITIALIZED` + is the only value `< INITIALIZED`, the previously generated error block became + unreachable and is no longer emitted.) +- The suite-cap `_final` body itself silent-returns on the same two + conditions (unallocated state array, or `== UNREGISTERED` for this instance). + After the first call's last-to-leave block deallocates `ccpp_suite_state`, + the unallocated state *is* the normal post-final condition — so on a + single-instance host the second call would otherwise trip a misleading + "`ccpp_register` has not been called" error. + +`_init` is intentionally *not* made idempotent on unallocated — there, +the unallocated state really does mean "you forgot `ccpp_register`", and +emitting an error is the correct behavior. + +The other physics phases (`init`, `timestep_init`, `run`, `timestep_final`) are +unchanged — they still hard-error with `errflg=1` on any state mismatch. + +--- + +## 9. Real-world example: CCPP Single Column Model (SCM) + +*Source:* `EXT/ccpp-scm/` — uses `ccpp-prebuild`. + +The SCM is a horizontally degenerate model (always `im = 1`, no OpenMP threading) but +it compiles the largest set of suites in the CCPP ecosystem, making it the most complete +real-world picture of what prebuild must handle. + +**Scale:** 63 suites, 257 scheme files (137 scheme entries in config, many containing +multiple modules), 300 generated cap files, ~1,200+ host model variables, ~550 optional +(conditionally active) variables. + +--- + +### 9.1 The `TYPEDEFS_NEW_METADATA` bridge — the DDT accessor map + +This is the most important SCM-specific configuration. It maps each DDT type name to the +Fortran expression used to access an instance of that type from the host model's top-level +scope. It is what allows the code generator to convert a `local_name` like `tgrs` (declared +inside `GFS_statein_type`) into the cap argument expression +`physics%Statein%tgrs(...)`. + +```python +TYPEDEFS_NEW_METADATA = { + 'GFS_typedefs': { + 'GFS_diag_type' : 'physics%Diag', + 'GFS_control_type' : 'physics%Model', + 'GFS_cldprop_type' : 'physics%Cldprop', + 'GFS_tbd_type' : 'physics%Tbd', + 'GFS_sfcprop_type' : 'physics%Sfcprop', + 'GFS_coupling_type': 'physics%Coupling', + 'GFS_statein_type' : 'physics%Statein', + 'GFS_radtend_type' : 'physics%Radtend', + 'GFS_grid_type' : 'physics%Grid', + 'GFS_stateout_type': 'physics%Stateout', + 'GFS_typedefs' : '', + }, + 'CCPP_typedefs': { + 'GFS_interstitial_type': 'physics%Interstitial(cdata%thrd_no)', + 'CCPP_typedefs' : '', + }, + 'scm_type_defs': { + 'physics_type': 'physics', + 'scm_type_defs': '', + }, + 'ccpp_types': { + 'ccpp_t' : 'cdata', + 'ccpp_types': '', + 'MPI_Comm': '', + }, + # ... plus 8 more entries for physics-side modules (machine, radsw_param, etc.) +} +``` + +**How it works:** For a variable with `local_name = tgrs` declared in `GFS_statein_type`, +the generator looks up `'GFS_statein_type'` in the map, finds `'physics%Statein'`, and +constructs the target as `physics%Statein%tgrs`. For the thread-indexed interstitial DDT, +`physics%Interstitial(cdata%thrd_no)%` is produced automatically. + +This dictionary is the **entire** mechanism by which the prebuild bridge converts flat +metadata into correct DDT-member accessor expressions. It is a hand-maintained workaround +that the redesigned generator must **eliminate**: all information needed to derive these +accessor expressions is already present in the CCPP metadata, provided the metadata storage +model is designed correctly to capture the DDT hierarchy and instance/thread indexing. + +--- + +### 9.2 Host model DDT structure + +``` +! Module-level variables accessible globally: +physics (type physics_type, from module scm_type_defs) +cdata (type ccpp_t, from module ccpp_types) +one (integer parameter = 1, from module ccpp_types) + +! physics_type contains: +physics%Model → GFS_control_type (control parameters: integers, logicals, 1D arrays) +physics%Statein → GFS_statein_type (input atmospheric state: 2D/3D real arrays) +physics%Stateout → GFS_stateout_type (output tendencies) +physics%Sfcprop → GFS_sfcprop_type (surface properties: 2D real arrays) +physics%Coupling → GFS_coupling_type (coupling fields) +physics%Grid → GFS_grid_type (grid geometry) +physics%Tbd → GFS_tbd_type (to-be-determined / miscellaneous) +physics%Cldprop → GFS_cldprop_type (cloud microphysics properties) +physics%Radtend → GFS_radtend_type (radiation tendencies) +physics%Diag → GFS_diag_type (diagnostic output arrays) +physics%Interstitial(1:thrd_cnt) → GFS_interstitial_type (per-thread scratch space) +``` + +The interstitial DDT is an array indexed by thread number. Even though the SCM is +single-threaded, all caps use `physics%Interstitial(cdata%thrd_no)` (i.e., index 1). +This is the pattern that enables OpenMP parallelism in the full UFS models. + +--- + +### 9.3 The horizontal dimension in the SCM + +The SCM uses a **chunked** horizontal loop even though `im = 1`. The chunk mechanism is: + +```fortran +chunk_begin = physics%Model%chunk_begin(cdata%chunk_no) +chunk_end = physics%Model%chunk_end(cdata%chunk_no) +``` + +All 2D and 3D array slice expressions in caps use this pattern: +```fortran +physics%Statein%tgrs(chunk_begin:chunk_end, one:levs) +``` + +In the SCM, `chunk_begin = chunk_end = 1` always, but the pattern is general enough for +multi-column models. The `one` lower bound (a named integer constant = 1) is a framework +convention used consistently throughout all caps. + +--- + +### 9.4 Four categories of local variables in group caps + +Every group cap generates four categories of local variable declarations before its scheme +calls: + +**Category 1 — Loop bounds and scalars (always present):** +```fortran +integer :: chunk_begin, chunk_end +integer :: levs +chunk_begin = physics%Model%chunk_begin(cdata%chunk_no) +chunk_end = physics%Model%chunk_end(cdata%chunk_no) +levs = physics%Model%levs +``` + +**Category 2 — Fixed-index extractions (tracer indices, surface-level slices):** + +For a tracer `qgrs(:,:,ntqv)`: +```fortran +! No local variable declared — the expression is used inline at the call site: +call scheme_run(qv = physics%Statein%qgrs(chunk_begin:chunk_end, one:levs, physics%Model%ntqv), ...) +``` + +For a surface-level slice `prsi(:,1)`: +```fortran +call scheme_run(prsi_sfc = physics%Statein%prsi(chunk_begin:chunk_end, 1), ...) +``` + +The fixed index may be a literal integer (`1`) or a runtime scalar variable from a DDT +field (`physics%Model%ntqv`). Both are inlined at the call site. + +**Category 3 — Optional variable pointer arrays:** + +One pointer-array type and one pointer-array variable are declared for each optional +variable. They are dimensioned by thread count: +```fortran +type :: real_kind_phys_rank2_ptr_arr_type + real(kind_phys), dimension(:,:), pointer :: p => null() +end type real_kind_phys_rank2_ptr_arr_type +type(real_kind_phys_rank2_ptr_arr_type), dimension(1:cdata%thrd_cnt) :: sfc_wts_1_ptr_array +``` + +Before each scheme call that uses the variable, the condition is evaluated and the pointer +either associated or left null: +```fortran +if (physics%Model%lndp_type /= 0) then + sfc_wts_1_ptr_array(cdata%thrd_no)%p => & + physics%Coupling%sfc_wts(chunk_begin:chunk_end, one:physics%Model%n_var_lndp) +end if +``` + +Passed to the scheme as a keyword argument: +```fortran +call gfs_surface_generic_pre_run(..., sfc_wts=sfc_wts_1_ptr_array(cdata%thrd_no)%p, ...) +``` + +After the call, the pointer is nullified: +```fortran +if (physics%Model%lndp_type /= 0) then + nullify(sfc_wts_1_ptr_array(cdata%thrd_no)%p) +end if +``` + +**Category 4 — Unit conversion local variables:** + +Not present in the SCM (GFS uses consistent SI units throughout). When present in other +models, a local array is declared, populated before the call, and passed as the argument: +```fortran +real(kind_phys) :: converted_var(chunk_begin:chunk_end) +converted_var(:) = physics%Statein%source_field(chunk_begin:chunk_end) * conversion_factor +call scheme_run(..., target_arg=converted_var, ...) +``` + +--- + +### 9.5 Array size checks + +Every array argument — mandatory or optional — has a size check immediately before the +scheme call. The check uses `size()` and computes the expected size from dimension variables: + +```fortran +! Mandatory variable — outer condition is always .true. +if (.true.) then + if (size(physics%Statein%tgrs(chunk_begin:chunk_end, one:levs)) /= & + (chunk_end-chunk_begin+1)*(levs-one+1)) then + write(cdata%errmsg, '(a,i8,a,i8)') & + 'Detected size mismatch for variable tgrs: expected ', expected, ' but got ', actual + ierr = 1 + return + end if +end if + +! Optional variable — outer condition mirrors the active= expression +if (physics%Model%lndp_type /= 0) then + if (associated(sfc_wts_1_ptr_array(cdata%thrd_no)%p)) then + if (size(sfc_wts_1_ptr_array(cdata%thrd_no)%p) /= expected_size) then + ...error... + end if + end if +end if +``` + +--- + +### 9.6 The `initialized(200)` array and instance management + +```fortran +logical, dimension(200), save :: initialized = .false. +``` + +`cdata%ccpp_instance` is a 1-based integer assigned to each independent CCPP state object. +In an ensemble, each ensemble member gets a different instance number (1–200). The `init_cap` +sets `initialized(cdata%ccpp_instance) = .true.` at the end of successful init. The +`run_cap` checks `if (.not. initialized(cdata%ccpp_instance))` and aborts with an error +if init was never called for that instance. The `final_cap` resets the flag to `.false.`. + +The value 200 is hardcoded — it is the maximum supported number of simultaneous model +instances. This could be made configurable. + +--- + +### 9.7 Suite and group cap hierarchy + +Three-level cap hierarchy: + +``` +ccpp_static_api.F90 (module ccpp_static_api) + → dispatches by suite_name + optional group_name + → owns physics, cdata, constants via module use + → calls suite-level caps: + +ccpp_scm_gfs_v16_cap.F90 (module ccpp_scm_gfs_v16_cap) + → aggregates arguments from all groups + → calls group caps in order per phase: + +ccpp_scm_gfs_v16_time_vary_cap.F90 (module ccpp_scm_gfs_v16_time_vary_cap) +ccpp_scm_gfs_v16_radiation_cap.F90 (module ccpp_scm_gfs_v16_radiation_cap) +ccpp_scm_gfs_v16_phys_ps_cap.F90 (module ccpp_scm_gfs_v16_phys_ps_cap) +ccpp_scm_gfs_v16_phys_ts_cap.F90 (module ccpp_scm_gfs_v16_phys_ts_cap) +``` + +Each level is a pure Fortran module. Argument passing is explicit keyword-argument style +at every level; no implicit global data (except in the static API, which uses `use`). + +--- + +### 9.8 Static API: module-level variable ownership + +The static API module uses all host-model modules and accesses their variables at module +scope. It does **not** take host data as subroutine arguments — instead it fills the +group cap arguments from its own module-use-associated variables: + +```fortran +module ccpp_static_api + use scm_type_defs, only: physics + use ccpp_types, only: cdata, one + use scm_physical_constants, only: con_g, con_pi, con_t0c, ... + use gfs_typedefs, only: ltp + use ccpp_scm_gfs_v16_cap, only: scm_gfs_v16_run_cap, ... + ... +contains + subroutine ccpp_physics_run(cdata, suite_name, group_name, ierr) + ! cdata passed in, others accessed from module scope + select case (to_lower(trim(suite_name))) + case ('scm_gfs_v16') + if (present(group_name)) then + select case (to_lower(trim(group_name))) + case ('phys_ps') + ierr = scm_gfs_v16_phys_ps_run_cap(one=one, physics=physics, cdata=cdata, ...) + ... + end select + else + ierr = scm_gfs_v16_run_cap(one=one, physics=physics, cdata=cdata, ...) + end if + case ('scm_gfs_v17_p8') + ... + end select + end subroutine +end module +``` + +This design means the static API file must be recompiled whenever any host-model module +changes (because it `use`s them), and it must be regenerated whenever suites change. +Its location in the **source tree** (not build tree) is a deliberate SCM design choice: +the file is committed to the repository as a generated artifact. + +--- + +### 9.9 Build system + +Prebuild runs at **cmake configure time** via `execute_process()`, before any compilation +starts. This is unusual but simplifies the cmake dependency graph. + +```cmake +execute_process( + COMMAND ccpp/framework/scripts/ccpp_prebuild.py + --config=ccpp/config/ccpp_prebuild_config.py + --suites=${CCPP_SUITES} + --builddir=${CMAKE_CURRENT_BINARY_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../.. + OUTPUT_FILE ${PROJECT_BINARY_DIR}/ccpp_prebuild.out + ERROR_FILE ${PROJECT_BINARY_DIR}/ccpp_prebuild.err +) +include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_CAPS.cmake) # → ${CAPS} +include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_SCHEMES.cmake) # → ${SCHEMES} +include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_TYPEDEFS.cmake) # → ${TYPEDEFS} +include(scm/src/CCPP_STATIC_API.cmake) # → ${API} +``` + +**Suite selection:** If `CCPP_SUITES` is not set by the user, a helper script +`suite_info.py` selects a compiler-appropriate subset. The full set of 63 suites is +used for production; subsets speed up development builds. + +--- + +### 9.10 Observations relevant to the redesign + +1. **`TYPEDEFS_NEW_METADATA` is a workaround that the redesign must eliminate.** The + DDT accessor information (which type lives at which accessor path) can be fully derived + from the CCPP metadata itself, given a well-designed metadata storage model. The + redesign must derive DDT accessor expressions automatically from the metadata rather + than requiring a separate hand-maintained dictionary. This is one of the primary + motivations for the new metadata storage design. + +2. **Three-level cap hierarchy (group → suite → static API) should be preserved.** + It provides clean separation: group caps are independently testable, suite caps + aggregate phases, the static API is the single host-callable entry point. + +3. **The static API's module-level `use` of host data is model-specific.** In models + where host data is not module-level (e.g., passed as subroutine arguments), the + static API pattern changes. The SCM is the simplest case because `physics` and `cdata` + are global module variables. + +4. **Instance and thread indexing are two orthogonal dimensions of host data access.** + Host model data uses two distinct indexing patterns that must be handled correctly: + + - **Regular state data** (Statein, Stateout, Sfcprop, etc.): dimensioned by instance + number — `physics%Statein(ccpp_instance_number)%array(1:horizontal_dimension, 1:vertical_dimension, ...)`. + In models supporting multiple in-memory model instances (ensemble), the top-level + DDT is an array indexed by `cdata%ccpp_instance`. + + - **Interstitial (per-thread scratch) data**: dimensioned by both instance and thread — + `physics%Interstitial(ccpp_instance_number, ccpp_thread_number)%array(1:horizontal_loop_extent, ...)`. + Critically, the horizontal dimension of interstitial arrays is sized to + `horizontal_loop_extent` (one OpenMP thread's chunk), not `horizontal_dimension` + (the full column count). `max_number_of_threads` instances are allocated per model + instance. Interstitial data can only be used during the **run phase** — this is a + known limitation of ccpp-prebuild that the redesign should address or at minimum + preserve explicitly. + +5. **Optional variable pointer arrays dimensioned by thread count** are the current + solution to thread-safe optional variable handling. This pattern is verbose (one + derived type + one array per optional variable per cap function) but correct. + The redesign could simplify this. + +6. **~550 optional variables in this model.** Optional/conditional variables are not + a corner case — they are a first-class feature. The redesign must handle them + efficiently and correctly. + +7. **Array size checks are debug-only and should not appear in the redesign by default.** + In prebuild they are only generated when the `--debug` flag is passed. The redesigned + generator should not produce them in normal mode — out-of-bounds access is caught at + runtime by compiler flags (e.g., `-fcheck=bounds` with gfortran, `-check bounds` with + ifort). The 12,991-line group cap is partly a consequence of generating these checks + unconditionally in the debug mode artifact examined here. + +8. **No unit conversions appear in GFS/SCM.** Unit conversion infrastructure must be + present in the redesign but the GFS physics package is self-consistent in units. + Unit conversions are more relevant for other host models. + +9. **The `one` constant** (integer parameter = 1) is passed as an explicit argument + everywhere and used as the lower bound in all array slices. This is a framework + convention. The redesign should decide whether this convention is preserved or + whether array lower bounds are handled differently. + +10. **Subcycles produce actual Fortran `do` loops inside the generated group cap.** + The loop from `1` to `cdata%loop_max` is generated directly in the cap function, + not left to the host model: + ```fortran + cdata%loop_max = 2 + do cdata%loop_cnt = 1, cdata%loop_max + call scheme_A_run(...) + if (ierr /= 0) return + call scheme_B_run(...) + if (ierr /= 0) return + end do + ``` + `cdata%loop_max` is set at the start of the subcycle block (from the `loop=` attribute + in the SDF XML) and `cdata%loop_cnt` is the current iteration counter, both visible + to schemes via the `ccpp_t` DDT. + +--- + +## 10. Real-world example: CAM-SIMA (capgen) + +*Source:* `EXT/cam-sima/` — uses `ccpp-capgen`. + +CAM-SIMA is the only model currently using capgen. It is still primarily a research model. +Unlike the SCM it uses a full 3D grid with OpenMP parallelism, but exposes host model +data as flat module variables rather than DDTs in the metadata layer. This example reveals +both what capgen can do and where it fundamentally fails. + +**Scale:** 1 suite (`cam7`), 2 run groups (`physics_before_coupler`, `physics_after_coupler`), +~75 scheme calls, 18 host `.meta` files, 893-line host cap, 2865-line suite cap. + +--- + +### 10.1 Suite structure + +`suite_cam7.xml` has two groups and no subcycles: + +| Group | Schemes (approx.) | Purpose | +|---|---|---| +| `physics_before_coupler` | 52 scheme calls | Cloud fraction, energy checks, dry adiabatic adjustment, Zhang-McFarlane deep convection full cycle, constituent tendency application | +| `physics_after_coupler` | ~20 scheme calls | Tropopause diagnostics, gravity wave drag (7 parameterizations + diagnostics), tendency application, energy consistency | + +CCPP phases in use: register, initialize, timestep_initial, run (per group), timestep_final, finalize. + +--- + +### 10.2 Host model variable structure + +**18 host `.meta` files, all of type `module`.** There are no `host` or `ddt` table types +anywhere. All host variables are flat scalars or arrays in Fortran modules. + +CAM-SIMA does **not** expose its physics DDTs (`phys_state`, `phys_tend`, `cam_in`, etc.) +through metadata. These types exist in `physics_types.F90` but have no `.meta` file. +The generated host cap accesses them directly via `use physics_types, only: phys_state, ...` +and passes individual DDT members as flat keyword arguments: +```fortran +! In cam_ccpp_cap.F90 — direct access to non-metadataized DDT members: +call cam7_physics_before_coupler(..., pint=phys_state%pint, t=phys_state%t, & + dtdt_total=phys_tend%dtdt_total, landfrac=cam_in%landfrac, ...) +``` + +This means capgen has no knowledge of how host data is structured. The host cap is +partly machine-generated and partly depends on manually wiring non-metadataized sources. +**This is a fundamental architectural gap** — changes to `physics_types` are invisible +to the framework. + +**Key host variables by module:** + +| Module | Key variables | +|---|---| +| `physics_grid` | `columns_on_task` (horizontal_dimension), `col_start`, `col_end`, lat, lon, area | +| `vert_coord` | `pver` (vertical_layer_dimension), `pverp` (vertical_interface_dimension) | +| `physconst` | ~35 physical constants, all `protected = True` | +| `cam_constituents` | `num_advected` (count of advected tracers) | +| `spmd_utils` | `mpicom`, `masterproc`, `npes`, `iam` | + +No instance indexing (`physics(1)`) and no thread-indexed DDTs appear — CAM-SIMA uses +a fundamentally different data model from the GFS/SCM stack. + +--- + +### 10.3 The two-cap architecture + +Capgen generates two distinct Fortran files: + +**`cam_ccpp_cap.F90` — the host cap (893 lines)** +- Module `cam_ccpp_cap` +- Imports non-metadataized host variables directly via `use physics_types`, `use physconst`, etc. +- Manages the constituent object (`ccpp_model_constituents_t`) — registration, initialization, gather/scatter, index lookup +- Public subroutines: `cam_ccpp_physics_run`, `cam_ccpp_physics_initialize`, etc. — the entry points the host model calls +- Dispatches to the suite cap, passing ~61–76 flat keyword arguments + +**`ccpp_cam7_cap.F90` — the suite cap (2865 lines)** +- Module `ccpp_cam7_cap` +- No host-specific imports — knows nothing about `physics_types`, `phys_state`, etc. +- All arguments are flat scalars and arrays, fully matched to metadata standard_names +- Contains all scheme calls, suite-level persistent variables, local temporaries, state machine +- The suite cap could in principle be used with any host model that provides the same standard names + +This two-cap split is **architecturally correct**: it separates host-specific binding +from physics-neutral dispatch. The redesign should preserve this separation. + +--- + +### 10.4 The flat-field argument problem — concrete evidence + +The run-phase subroutines expose the core problem with capgen's approach directly: + +```fortran +subroutine cam7_physics_before_coupler(errflg, errmsg, col_start, col_end, pver, dtime, & + gravit, pint, te_ini_dyn, teout, amiroot, iulog, ptend_s, temp, dtdt_total, cpair, & + lagrang, layer_surf, layer_toa, interface_surf, interface_toa, ncnst, piln, pmid, pdel, & + rpdel, qv, carr, cprops, rair, zvir, zi, zm, cp_or_cv_dycore, u, v, pintdry, phis, & + te_cur_phys, te_cur_dyn, tw_cur, latice, latvap, energy_formula_physics, & + energy_formula_dycore, cappa, q_tend, const_tend, qmin, pverp, cpwv, cpliq, rh2o, lat, & + long, pblh, mcon, tpert, dlf, rprd, ql, rliq, landfrac, cpair3, ttend_dp, tmelt, & + top_lev, ke, ke_lnd, cldfrc, domomtran, momcu, momcd, il1g, nstep, & + dudt_total, dvdt_total, fracis, dpdry, ps) +``` + +**61 dummy arguments for one group cap.** `physics_after_coupler` has 76. These are +individual flat arrays and scalars — no DDT in sight. This is exactly the problem that +three developers failed to fix: in the GFS/UFS context, this would be 1,200+ arguments. +The GFS physics stack simply cannot be connected to capgen in its current form. + +In contrast, the prebuild equivalent for the same data would pass `physics` (one DDT argument) +and `cdata` — two arguments covering hundreds of variables. + +--- + +### 10.5 Suite-level persistent variables — the framework-owned data pattern + +The suite cap allocates and owns arrays that persist across group calls within a timestep. +These are allocated in `cam7_initialize` and deallocated in `cam7_finalize`: + +```fortran +! Suite-level persistent (allocated in initialize, freed in finalize): +real(kind_phys), allocatable :: windu_tend(:,:) ! GW drag u-tendency accumulator +real(kind_phys), allocatable :: windv_tend(:,:) ! GW drag v-tendency accumulator +real(kind_phys), allocatable :: scaling_dycore(:,:) ! energy scaling factor +real(kind_phys), allocatable :: tend_te_tnd(:) ! energy tendency accumulator +real(kind_phys), allocatable :: tend_tw_tnd(:) ! water tendency accumulator +real(kind_phys), allocatable :: temp_ini(:,:) ! temperature saved at timestep start +real(kind_phys), allocatable :: z_ini(:,:) ! height saved at timestep start +real(kind_phys), allocatable :: flx_vap(:), flx_cnd(:), flx_ice(:), flx_sen(:) +logical, allocatable :: doconvtran(:) ! per-constituent convection flag +type(coords1d) :: p ! pressure coordinate DDT for GW drag +``` + +These are physics-internal variables — the host model does not know about them, does not +own them, and does not need to. This is the capgen "data ownership" model: the suite cap +is the data owner for variables that only matter within the physics. + +**This pattern is correct and desirable.** The complexity in capgen comes not from the +concept but from how these variables are discovered during analysis (scope-chain promotion) +and passed around (via VarDictionary). The redesign needs a simpler mechanism to achieve +the same result: statically enumerate physics-internal variables during analysis and have +the suite cap own them as named allocatables. + +During the run phase, suite-level persistent arrays are subsetted when passed to schemes: +```fortran +call gw_common_run(..., windu_tend=windu_tend(col_start:col_end, 1:pver), ...) +``` + +Run-phase local temporaries (e.g., `cape`, `cme`, `mu`, `md`) are allocated at function +entry and deallocated at exit: +```fortran +allocate(cape(col_start:col_end)) +... +call zm_convr_run(..., cape=cape, ...) +... +deallocate(cape) +``` + +These temporaries use `col_start` as the lower bound so that assumed-shape dummy arguments +in schemes see a 1-based array — a subtle but important detail. + +--- + +### 10.6 Horizontal chunking model + +CAM-SIMA uses `col_start`/`col_end` (passed as arguments to every run subroutine) to +define the current horizontal chunk: + +```fortran +ncol = col_end - col_start + 1 +``` + +Schemes declare `horizontal_loop_extent` and receive `ncol`. The horizontal dimension +in the host (storage dimension) is `columns_on_task`. The subsetting from storage to +loop extent happens at the boundary between host cap and suite cap — the host cap +passes the right subsections: + +```fortran +! In cam_ccpp_cap.F90: +call cam7_physics_before_coupler(..., col_start=col_start, col_end=col_end, & + pmid=phys_state%pmid, ...) ! full arrays passed; suite cap subsets internally +``` + +Inside the suite cap, persistent arrays are subsetted explicitly when passed to schemes: +```fortran +windu_tend(col_start:col_end, 1:pver) +``` +Local temporaries allocated as `allocate(cape(col_start:col_end))` are already +correctly sized and passed as assumed-shape `(:)`. + +--- + +### 10.7 State machine + +The suite cap has a character module variable tracking lifecycle state: + +```fortran +character(len=16) :: ccpp_suite_state = 'uninitialized' +``` + +Transitions: `uninitialized` → register → `uninitialized` → initialize → `initialized` +→ timestep_initial → `in_time_step` → (run, no state change) → timestep_final → +`initialized` → finalize → `uninitialized`. + +Each phase entry point checks the expected prior state: +```fortran +if (trim(ccpp_suite_state) /= 'in_time_step') then + errflg = 1 + write(errmsg, '(3a)') "Invalid initial CCPP state, '", trim(ccpp_suite_state), & + "' in cam7_physics_before_coupler" + return +end if +``` + +Non-run phases also include an OpenMP thread guard: +```fortran +#ifdef _OPENMP + if (omp_get_thread_num() > 1) then + errflg = 1 + errmsg = "Cannot call initialize routine from a threaded region" + return + end if +#endif +``` + +The state machine is simple, complete, and useful. The redesign should preserve it. + +--- + +### 10.8 Constituent variable handling + +CAM-SIMA demonstrates the full constituent lifecycle: + +```fortran +! In cam_ccpp_cap.F90: +type(ccpp_model_constituents_t), target :: cam_constituents_obj + +! Registration (scheme-declared constituents): +call suite_cam7_constituents_num_consts(num_consts) +call suite_cam7_constituents_const_name(iconst, const_name) +call cam_constituents_obj%new_field(const_name, ...) + +! Initialization (host-declared constituents like water vapor): +cam_model_const_stdnames(1) = "water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water" +call cam_constituents_obj%new_field(cam_model_const_stdnames(1), ...) + +! Per-timestep gather from host: +call cam_ccpp_gather_constituents(phys_state%q, ...) + +! Passing to suite cap: +call cam7_physics_before_coupler(..., + qv = cam_constituents_obj%vars_layer(:, :, cam_model_const_indices(1)), + carr = cam_constituents_obj%vars_layer, + cprops = cam_constituents_obj%const_metadata, ...) + +! Per-timestep scatter back to host: +call cam_ccpp_update_constituents(phys_state%q, ...) +``` + +The suite cap sees constituents as: +- `carr(:,:,:)` — the full rank-3 constituent array (ncol, nlev, ncnst) +- `qv(:,:)` — water vapor slice extracted in the host cap: `cam_constituents_obj%vars_layer(:,:,cam_model_const_indices(1))` +- `cprops(:)` — array of `ccpp_constituent_prop_ptr_t` metadata objects +- `doconvtran(1:ncnst)` — suite-level logical array set by scheme init indicating which constituents are convected + +This constituent API is sophisticated and worth preserving or improving in the redesign. + +--- + +### 10.9 Known defects in the capgen output + +**Repeated scheme init/final calls.** Capgen generates one init call per occurrence of +a scheme name in the XML, without deduplication: +- `qneg_init` called 5 times (once per `qneg` entry in the suite XML) +- `qneg_timestep_final` called 5 times +- `check_energy_chng_init` called twice +- `save_ttend_from_convect_deep_timestep_init` called 3 times + +If these routines have internal state, allocations, or side effects, this is a correctness +defect. The redesign must deduplicate init/final calls by unique scheme name. + +**Unit conversion embedded silently in the cap.** Before `zm_conv_convtran_run`: +```fortran +dpdry_local(:,1:pver) = 1.0E-2_kind_phys * dpdry(:,1:pver) ! Pa → hPa +``` +This is generated from the metadata units mismatch but appears as an opaque transform +in the cap. The redesign should make this visible (e.g., a comment naming the standard +name, the source units, and the target units). + +--- + +### 10.10 Build system — capgen invocation + +Capgen is invoked from Python (`cam_autogen.py`), not from cmake: + +```python +from ccpp_capgen import capgen +capgen_db = capgen(run_env, return_db=True) +``` + +This is a programmatic API call, not a subprocess. The `CCPPDatabaseObj` returned +(`capgen_db`) is then used directly in Python to query scheme lists, constituent names, +and file paths — avoiding the datatable XML query step that cmake-based invocations need. + +Output files consumed by the build: +- `cam_ccpp_cap.F90` — compiled into the atmosphere component +- `ccpp_cam7_cap.F90` — compiled into the atmosphere component +- `ccpp_kinds.F90` — compiled into the atmosphere component +- Utility files from `ccpp_framework/src/` (copied to build dir) +- `ccpp_datatable.xml` — queried by the build system for file lists + +--- + +### 10.11 Observations relevant to the redesign + +1. **The two-cap split (host cap + suite cap) is the right architecture.** It cleanly + separates host-specific binding from physics-neutral dispatch. The redesign must + preserve this. + +2. **Flat-field arguments in the suite cap are the critical failure.** 61–76 dummy + arguments per run subroutine is already large for a research model; for UFS/GFS + with 1,200+ variables it is completely infeasible. The redesign must pass DDTs. + +3. **The CAM-SIMA host does not use DDTs in metadata.** All host variables are flat + module variables. This is a fundamentally different host model architecture from + GFS/SCM. The redesign must support both styles: flat-module hosts (CAM-SIMA) and + deep-DDT hosts (GFS/SCM). + +4. **Non-metadataized variables hardwired into the host cap is a serious gap.** + `phys_state`, `phys_tend`, `cam_in` from `physics_types` have no `.meta` files. + The host cap accesses them directly. This means the framework cannot verify or + track these variables. The redesign should either require full metadata coverage + or have an explicit mechanism for declaring non-metadataized pass-through variables. + +5. **Suite-level persistent variables (framework-owned data) work well in practice.** + `windu_tend`, `scaling_dycore`, `temp_ini`, etc. are owned by the suite cap, invisible + to the host, and persist across group calls. This is the right pattern for + physics-internal state. The redesign needs this but with a simpler discovery mechanism + than capgen's scope-chain promotion. + +6. **Deduplicate init/final calls.** The redesign must deduplicate `_init`, `_finalize`, + `_timestep_init`, and `_timestep_final` calls by unique scheme name (not by occurrence + in the XML). + +7. **The constituent API in the host cap is comprehensive.** The `ccpp_model_constituents_t` + object with its register/init/gather/scatter/index API is sophisticated and should be + preserved or improved. + +8. **The suite-variables introspection subroutine** (`ccpp_physics_suite_variables`, + enumerating 83 standard names as inputs/outputs) is a useful capability for build + system integration and should be in the redesign. + +9. **The programmatic Python API** (`capgen(run_env, return_db=True)`) is valuable + for hosts like CAM-SIMA that invoke the generator from Python. The redesign should + support both CLI and programmatic invocation. + +10. **Unit conversions must be annotated in the generated cap**, not silently embedded + as magic-number multiplications. A comment with source units, target units, and the + standard name involved is the minimum. + +11. **The horizontal chunking model** (`col_start`/`col_end` as explicit arguments, + `ncol = col_end - col_start + 1` computed at entry) works and is clean. Suite-level + persistent arrays are allocated full-size and subsetted at call sites. + +12. **No optional variables in this model.** CAM-SIMA does not exercise optional/active + variable handling. This feature must be in the redesign but is not demonstrated here. + +--- + +## 11. Real-world example: UFS Weather Model (prebuild) + +The UFS Weather Model is the most complex and production-critical of the three examples. +It is a fully-coupled, 3-D operational NWP model. The CCPP physics is used in the +atmospheric component (`UFSATM`). Unlike the SCM (column model, process-split only) and +CAM-SIMA (capgen, flat-field arguments), UFS uses prebuild in a 3-D blocked/threaded +configuration that is architecturally distinct from both prior examples. + +The two suites analyzed here are: +- `FV3_GFS_v17_coupled_p8` — the primary operational GFS suite +- `FV3_GFS_v17_coupled_p8_ugwpv1` — a variant replacing `unified_ugwp` with `ugwpv1` + +The ugwpv1 suite is structurally identical to the base suite except for the `phys_ps` +group (4 extra scheme calls), so all observations below apply to both. + +--- + +### 11.1 Suite structure + +The primary suite has 5 groups: + +| Group | Subcycles | Scheme calls | Phase called | +|-------|-----------|-------------|--------------| +| `time_vary` | 1 | 4 | timestep_init (domain-level, no blocking) | +| `radiation` | 1 | 8 | run (block/thread loop) | +| `phys_ps` | 3 (loop=1, loop=2, loop=1) | 21 | run (block/thread loop) | +| `phys_ts` | 3 (loop=1, loop=1, loop=1) | 12 | run (block/thread loop) | +| `stochastics` | 1 | 2 | run (block/thread loop) | + +The `time_vary` group is the only one called at timestep_init/finalize. All other groups +are called from the run phase via the OpenMP blocked loop. This is a fundamentally +different usage pattern from SCM (which runs everything sequentially) and CAM-SIMA +(which has no run phase at all for the groups analyzed). + +The `phys_ps` group has a surface iteration subcycle with `loop="2"`, which generates an +actual Fortran `do` loop in the cap body: +```fortran +! Start of next subcycle +cdata%loop_max = 2 +do cdata%loop_cnt = 1, cdata%loop_max + ! ... sfc_diff, sfc_nst, noahmpdrv, sfc_land, sfc_cice, sfc_sice ... +end do +``` + +--- + +### 11.2 Cap hierarchy and scale + +The three-level hierarchy is preserved from prebuild: + +``` +ccpp_static_api.F90 (627 lines) ← suite+group name dispatch + ↓ +ccpp_fv3_gfs_v17_coupled_p8_cap.F90 (363 lines) ← calls all group caps in order + ↓ +ccpp_fv3_gfs_v17_coupled_p8_time_vary_cap.F90 (1404 lines) +ccpp_fv3_gfs_v17_coupled_p8_radiation_cap.F90 (967 lines) +ccpp_fv3_gfs_v17_coupled_p8_phys_ps_cap.F90 (4226 lines) ← 200 optional ptr arrays +ccpp_fv3_gfs_v17_coupled_p8_phys_ts_cap.F90 (1953 lines) +ccpp_fv3_gfs_v17_coupled_p8_stochastics_cap.F90 (443 lines) +``` + +The ugwpv1 variant generates another 10,220 lines of largely redundant code (identical +caps with one suite-name prefix change and minor scheme-list differences). Total for both +suites: 18,333 lines of generated Fortran. + +This redundancy is a key motivation for the redesign: suite variants that share groups +should not regenerate identical cap code. The redesign should support group-level cap +sharing across suite variants. + +--- + +### 11.3 Host model DDT structure + +All host data lives in `CCPP_data.F90` as module-level `save, target` variables: + +```fortran +type(GFS_control_type) :: GFS_control ! config/control +type(GFS_statein_type) :: GFS_statein ! atmospheric state in +type(GFS_stateout_type) :: GFS_stateout ! atmospheric state out +type(GFS_grid_type) :: GFS_grid ! grid geometry +type(GFS_tbd_type) :: GFS_tbd ! temporal interp data +type(GFS_cldprop_type) :: GFS_cldprop ! cloud properties +type(GFS_sfcprop_type) :: GFS_sfcprop ! surface properties +type(GFS_radtend_type) :: GFS_radtend ! radiation tendencies +type(GFS_coupling_type) :: GFS_coupling ! coupling fields +type(GFS_diag_type) :: GFS_intdiag ! diagnostics +type(GFS_interstitial_type), allocatable (:) :: GFS_interstitial ! scratch, per thread +``` + +Plus three `ccpp_t` instances for different levels of parallelism (see §11.5). + +This is structurally similar to the SCM's `physics` DDT hierarchy, but with one key +difference: all DDTs are at the same flat level rather than nested (no `physics%Statein`, +only `GFS_statein`). Each DDT maps to a distinct functional role. + +The `GFS_typedefs.F90` file (not auto-generated) defines all DDT types along with ~30 +physical constants (`con_pi`, `con_g`, `con_rd`, etc.) that also appear in the metadata. + +--- + +### 11.4 DDT arguments in the cap chain + +The static API imports all DDTs and physical constants from `CCPP_data` and `GFS_typedefs` +via `use` statements, then passes them as named arguments to group cap functions. This is +the full DDT-argument pattern that prebuild implements: + +```fortran +! In ccpp_static_api.F90: +use ccpp_data, only: gfs_control, gfs_statein, gfs_sfcprop, ... +use gfs_typedefs, only: con_pi, con_g, con_rd, ... + +ierr = fv3_gfs_v17_coupled_p8_phys_ps_run_cap( & + one=one, gfs_control=gfs_control, cdata=cdata, & + gfs_statein=gfs_statein, gfs_sfcprop=gfs_sfcprop, & + con_g=con_g, con_pi=con_pi, ... & + gfs_interstitial=gfs_interstitial) +``` + +The group cap receives these as typed `intent(*), target` dummy arguments and uses them +directly to construct call-site subsections. This means **the group cap is fully portable +— it does not use any host module directly**, only what it receives as arguments. + +The `target` attribute is required because the cap creates pointer sections of these DDTs +(array subsections via pointer assignment) when handling optional variables. + +--- + +### 11.5 The dual cdata architecture + +UFS uses two distinct sets of `ccpp_t` handles with different scopes: + +**Domain-level (`cdata_domain`)**: Used for non-run phases (init, finalize, time_vary +timestep_init/finalize). Called once per step, no blocking: +```fortran +cdata_domain%blk_no = 1; cdata_domain%chunk_no = 1 +cdata_domain%thrd_no = 1; cdata_domain%thrd_cnt = 1 +``` + +**Block/thread-level (`cdata_block(nb, nt)`)**: Used for run phase (radiation, phys_ps, +phys_ts, stochastics). Allocated as a 2-D array `(1:nblks, 1:nthrdsX)` where `nthrdsX` +accounts for non-uniform last-block sizing: +```fortran +cdata_block(nb,nt)%blk_no = nb +cdata_block(nb,nt)%chunk_no = nb ! block number = chunk number +cdata_block(nb,nt)%thrd_no = nt +cdata_block(nb,nt)%thrd_cnt = nthrdsX +``` + +The redesign must support this dual cdata usage: a single `cdata` handle for domain-level +phases and a 2-D array of handles for blocked run phases. + +--- + +### 11.6 OpenMP threading model + +Non-run phases allow internal threading in physics schemes: +```fortran +GFS_control%nthreads = nthrds ! all N threads available to physics +call ccpp_physics_timestep_init(cdata_domain, ...) +``` + +Run phase uses all threads for blocking, so physics must not spawn additional threads: +```fortran +GFS_control%nthreads = 1 ! no internal threading allowed +!$OMP parallel num_threads(nthrds) ... +!$OMP do schedule(dynamic,1) +do nb = 1, nblks + call GFS_Interstitial(nt)%create(ixs=chunk_begin(nb), ixe=chunk_end(nb), model=GFS_control) + call ccpp_physics_run(cdata_block(nb,nt), group_name="phys_ps", ...) + call GFS_Interstitial(nt)%destroy(GFS_control) +end do +!$OMP end do +!$OMP end parallel +``` + +The `nt = omp_get_thread_num()+1` pattern (1-based thread index) is used throughout. +Each thread owns one `GFS_Interstitial(nt)` and one `cdata_block(nb,nt)` per block +iteration. The dynamic schedule means different threads process different blocks at +different times, which is why the interstitial must be created/destroyed per-iteration +rather than pre-allocated per-thread. + +--- + +### 11.7 Horizontal dimension: the chunk_begin/chunk_end pattern + +For non-run phases, the full horizontal dimension is used at every call site: +```fortran +tgrs(one:gfs_control%ncols, one:gfs_control%levs) +``` + +For run phases, the chunk range is looked up from the control DDT using the block number: +```fortran +tgrs(gfs_control%chunk_begin(cdata%chunk_no) : gfs_control%chunk_end(cdata%chunk_no), & + one:gfs_control%levs) +``` + +The chunk size (horizontal extent `im`) is retrieved as: +```fortran +im = gfs_control%blksz(cdata%blk_no) +``` + +`blksz(nb)` handles **non-uniform block sizes**: the last block may be smaller than the +others if the domain size is not divisible by the number of blocks. The `chunk_begin`/ +`chunk_end` arrays (indexed by chunk number = block number) give the global offset range. + +This is a cleaner pattern than SCM's `chunk_begin`/`chunk_end` as explicit dummy +arguments, because UFS looks them up from the already-passed `gfs_control` DDT. + +**Critical implication for the redesign**: The subsetting pattern `(chunk_begin:chunk_end)` +appears at every single array call site in the run phase — literally hundreds of times in +the phys_ps cap alone. This boilerplate is generated by prebuild from the metadata. In +the redesign, this subsetting must remain at the call site (not higher up) to allow each +thread to process its own chunk independently. + +--- + +### 11.8 The GFS_interstitial — pointer-based scratch DDT + +`GFS_interstitial_type` (defined in `CCPP_typedefs.F90`) is a DDT where **every field is +a pointer**, initialized to null: +```fortran +type GFS_interstitial_type + real(kind_phys), pointer :: adjsfculw_land(:) => null() + real(kind_phys), pointer :: del(:,:) => null() + ! ... ~200+ pointer fields +end type +``` + +This is dramatically different from the SCM's interstitial (which is a regular allocatable +DDT allocated once per thread at startup). The UFS interstitial is: +1. **Created** (`GFS_Interstitial(nt)%create(ixs, ixe, model)`) before each block — this + allocates all required fields to the chunk size `ixe-ixs+1` +2. **Reset** (`GFS_Interstitial(nt)%reset(model)`) to zero before radiation and phys_ps +3. **Destroyed** (`GFS_Interstitial(nt)%destroy(model)`) after each block — deallocates + +This design exists because different blocks (especially the last block) can have different +sizes. Pre-allocating to the maximum size wastes memory at scale; per-block allocation +ensures exact sizing. The pointer-based design also allows the `create()` method to +selectively allocate only the fields needed for the current physics configuration. + +In the caps, the interstitial is accessed as: +```fortran +gfs_interstitial(cdata%thrd_no)%del(chunk_begin:chunk_end, one:levs) +``` + +The interstitial array is 1-D (indexed by thread, not by `(instance, thread)` as in SCM). +This works because UFS has only one model instance at runtime — no ensemble-in-memory. + +--- + +### 11.9 Optional variables — the pointer array pattern at scale + +The phys_ps run cap has **200 optional pointer arrays** in its local variable section. +Each looks like: +```fortran +type :: real_kind_phys_rank1_ptr_arr_type + real(kind_phys), dimension(:), pointer :: p => null() +end type real_kind_phys_rank1_ptr_arr_type +type(real_kind_phys_rank1_ptr_arr_type), dimension(1:cdata%thrd_cnt) :: sfc_wts_1_ptr_array +``` + +Usage pattern (consistent with SCM but with threading dimension): +```fortran +if (gfs_control%lndp_type /= 0) then + sfc_wts_1_ptr_array(cdata%thrd_no)%p => & + gfs_coupling%sfc_wts(chunk_begin:chunk_end, one:gfs_control%n_var_lndp) +end if +! ... scheme call ... +if (gfs_control%lndp_type /= 0) then + nullify(sfc_wts_1_ptr_array(cdata%thrd_no)%p) +end if +``` + +The array is dimensioned by `cdata%thrd_cnt` (total thread count) and indexed by +`cdata%thrd_no` (current thread number). This handles the threaded run phase where +multiple threads are simultaneously executing the same run cap function with different +chunk ranges. Each thread independently associates and nullifies its own pointer slot. + +200 optional variables in `phys_ps` alone. This is the regime for which the SCM had ~550 +total optional vars — confirming that operational 3-D GFS physics is heavily optional-var +driven. The design is sound but generates enormous boilerplate. + +A key observation: the type definition for each pointer wrapper (`integer_..._ptr_arr_type`, +`real_kind_phys_rank1_ptr_arr_type`, etc.) is **re-declared inside every single function +that needs it**. This results in duplicate type definitions across all group caps. The +redesign should define these wrapper types once in a shared module. + +--- + +### 11.10 Physical constants as metadata variables + +The UFS static API has an extensive USE list of physical constants from `gfs_typedefs`: +``` +con_pi, con_g, con_t0c, con_hfus, con_solr_2008, con_solr_2002, con_c, con_plnk, +con_boltz, con_rd, ltp, con_zero, con_rerth, con_p0, con_rv, con_cp, con_rgas, +con_amd, con_amw, con_avgd, con_hvap, con_eps, con_omega, con_fvirt, con_ttp, +con_thgni, con_epsm1, con_rog, con_rocp, con_tice, con_sbc, con_jcal, con_rhw0, +rlapse, rhowater, karman, con_1ovg, con_cliq, con_cvap, rainmin, con_epsm1 (30+ total) +``` + +These travel through the full chain: static API USE → suite cap argument → group cap +argument → scheme call argument. Each constant is declared as a separate scalar dummy +argument (`real(kind_phys), intent(in), target :: con_pi`) in every group cap that needs +it. + +This is correct but verbose. The redesign should consider whether constants should be +gathered into a dedicated DDT (e.g., `gfs_constants_type`) so the cap chain carries one +argument instead of 30. This would also eliminate the need to explicitly enumerate which +constants each group needs — they could all come along in the constants DDT. + +--- + +### 11.11 The `one` lower-bound anchor + +The integer constant `one = 1` (from `ccpp_types`) is passed as an explicit argument +throughout the UFS cap chain for the same reason as in SCM: it anchors lower array bounds +without triggering association-status issues: +```fortran +type(gfs_interstitial_type), intent(inout), target :: gfs_interstitial(one:) +tgrs(one:gfs_control%ncols, one:gfs_control%levs) +``` + +This pattern is ubiquitous and is a known prebuild idiom. + +--- + +### 11.12 No framework-owned persistent variables + +Unlike CAM-SIMA (which allocates scheme-persistent variables in the suite cap), the UFS +has no framework-owned persistent state in any cap. All persistent state lives in the host +DDTs (`GFS_tbd`, `GFS_sfcprop`, etc.). The interstitial DDT (`GFS_interstitial`) is purely +transient — created and destroyed each block. + +This is consistent with UFS's prebuild-based architecture. Whether framework-owned +persistent variables would be beneficial for UFS is an open question for the redesign. + +--- + +### 11.13 Build system and driver + +Prebuild is invoked from CMake (not programmatically) and generates: +- Group cap files (one per group × suites) +- Suite cap files (one per suite) +- `ccpp_static_api.F90` +- `CCPP_CAPS.cmake`, `CCPP_SCHEMES.cmake`, `CCPP_TYPEDEFS.cmake` — consumed by CMake to + enumerate files to compile + +The host driver (`CCPP_driver.F90`) is **hand-written**, not auto-generated. It owns the +OpenMP loop, the cdata allocation/setup, the interstitial create/destroy, and the +diagnostic bucket zeroing. This is a significant difference from CAM-SIMA where the +equivalent driver code is partially generated. In the redesign, this host driver code +should remain hand-written — it encodes model-specific threading and blocking decisions +that cannot be derived from metadata alone. + +--- + +### 11.14 Observations relevant to the redesign + +1. **The DDT-argument cap chain is fully validated at UFS scale.** Passing 10+ DDTs plus + 30+ scalar constants as named arguments through three cap levels works correctly in + production. The redesign must replicate this exactly. + +2. **The chunk_begin/chunk_end subsetting at call sites is non-negotiable.** Hundreds of + array sections per group cap. The generator must produce this from the metadata + `horizontal_dimension` standard name and the `active` flag for optional variables. + This is prebuild's core value at 3-D scale. + + *Design direction*: Rather than carrying `chunk_no` in cdata and having the cap look + up `gfs_control%chunk_begin(chunk_no)`, the redesign should pass + `horizontal_loop_begin` and `horizontal_loop_end` as explicit arguments directly to + `ccpp_physics_run()` (and analogous calls). This decouples the cap from knowing about + the host's internal chunk-lookup arrays. The host driver sets these for each block + iteration and passes them in; the cap uses them directly. + +3. **The domain-vs-block execution contexts must be supported, but the cdata object is + not necessarily the right mechanism.** The key information is: instance number, thread + number, horizontal_loop_begin, horizontal_loop_end, error flag/message. If all of + these are explicit named arguments to `ccpp_physics_*`, the cdata object becomes + redundant scaffolding. This is an open design question to be discussed separately, but + the UFS analysis shows that cdata carries exactly these values — the object is a + transport container, not a framework abstraction. + +4. **The `blksz` non-uniform block size is a first-class concern.** The generator must + produce `im = gfs_control%blksz(cdata%blk_no)` (or an equivalent `horizontal_loop_extent` + computed from the explicit begin/end) for the horizontal extent argument in run phases. + +5. **GFS_interstitial as a pointer-DDT is the correct design for 3-D models.** Creating + and destroying per block avoids memory waste from over-allocation to the maximum chunk + size. The pointer-based field design enables selective allocation. The redesign should + document this pattern and support it. (Whether the generator should emit the + `type(X_interstitial_type)` DDT definition itself or only the caps is TBD.) + +6. **200 optional pointer arrays in one group cap is manageable but the wrapper type + proliferation is not.** The 4 wrapper types (`integer_r1_ptr_arr_type`, + `real_r1_ptr_arr_type`, `real_r2_ptr_arr_type`, `character_len3_r1_ptr_arr_type`) + should be defined once in a shared module (e.g., `ccpp_types.F90`) and reused across + all caps, eliminating thousands of duplicate lines. + +7. **Physical constants as metadata variables must be gathered into a constants DDT.** + The redesign will collect all physics constants into a single `constants_type` DDT + (or equivalent), reducing 30+ individual scalar arguments in the cap chain to one + argument. This requires a metadata declaration mechanism for compound read-only + objects (i.e., constants do not need intent tracking the way state variables do). + +8. **No framework-owned persistent variables in UFS** confirms that this feature is + optional and model-specific. The redesign needs to support it (for CAM-SIMA-like + models) but should not force it on models that do not need it. + +9. **The host driver is correctly hand-written.** The OpenMP blocking, interstitial + lifecycle, diagnostic bucket management — these are model-specific decisions that + belong in the host driver, not in generated code. The redesign should not try to + generate the driver. + +10. **Suite variant cap redundancy is not a concern.** For research/development, multiple + suites are active simultaneously and generated code size doesn't matter. For + production, only one suite is compiled and used at a time. The redesign need not + prioritize eliminating redundant group cap code across suite variants. + +--- + +## 12. Real-world example: Navy NEPTUNE (prebuild, restricted) + +The NEPTUNE source code cannot be shared. The following is based on architectural +description provided by the lead developer. + +NEPTUNE uses `ccpp-prebuild` with the same GFS physics as UFS and nearly identical suites. +Its unique distinguishing feature is **multiple coexisting CCPP physics instances** — it +is the only model among the four examples that exercises this capability at runtime. + +--- + +### 12.1 Multiple instances — the N-dimensioned DDT array mechanism + +In NEPTUNE, the host model allocates N copies of all GFS DDTs as 1-D arrays indexed by +instance number: + +```fortran +type(GFS_sfcprop_type), allocatable :: gfs_sfcprop(1:N) +type(GFS_statein_type), allocatable :: gfs_statein(1:N) +type(GFS_stateout_type), allocatable :: gfs_stateout(1:N) +! ... all GFS DDTs dimensioned 1:N +type(GFS_control_type), allocatable :: gfs_control(1:N) +``` + +The static API imports these module-level arrays via `use` statements (same as UFS). +The instance selection happens at the call site inside the group cap, using +`cdata%ccpp_instance` as the array index: + +```fortran +call foo_run( & + tair = gfs_statein(cdata%ccpp_instance)%tair( & + gfs_control(cdata%ccpp_instance)%chunk_begin(cdata%chunk_no) : & + gfs_control(cdata%ccpp_instance)%chunk_end(cdata%chunk_no), & + 1:nvertical), & + ...) +``` + +Three things are happening simultaneously at each call-site array section: +1. **Instance selection**: `gfs_statein(cdata%ccpp_instance)` picks the correct DDT from + the N-element array +2. **Chunk subsetting**: `chunk_begin(chunk_no):chunk_end(chunk_no)` applies the run-phase + horizontal slice +3. **Vertical bound**: explicit `1:nvertical` + +This is the same pattern as UFS except the DDTs are 1-D arrays rather than scalars. +The generator must produce this instance-indexed subsetting when the host declares its +DDTs as arrays. + +--- + +### 12.2 What NEPTUNE tells us about `cdata%ccpp_instance` + +The `initialized(200)` array in every group cap (confirmed in both SCM and UFS caps) now +has its full motivation: it handles up to 200 simultaneous instances without requiring +per-instance cap code. The `cdata%ccpp_instance` value (1-based) is the runtime selector +into both the host DDT arrays and the `initialized` guard array. + +NEPTUNE is the reason `200` is not `1`. In single-instance models (UFS, SCM, CAM-SIMA) +`cdata%ccpp_instance` is always 1 and the N-dimensioned DDT arrays have `N=1`. + +--- + +### 12.3 Observations relevant to the redesign + +1. **Multiple instances require only one change at the call site**: inserting the instance + index at the correct dimension position. Everything else (chunking, optional variables, + threading) composes with this unchanged. + +2. **The instance dimension can appear anywhere in any host variable — not just as an + index into an array of DDTs.** A flat array `flat_field(1:ninstance, 1:nhoriz, 1:nvert)` + is equally valid; its call site becomes: + ```fortran + flat_field(instance_number, horiz_begin:horiz_end, 1:nvert) + ``` + The generator handles this by classifying each dimension by its declared standard name. + `instance_dimension` is a registered standard name (like `horizontal_dimension` and + `vertical_dimension`) — the generator knows its semantics regardless of where it + appears in the dimension list or whether the variable is a DDT array element or a + plain array. See §13.4 for the full dimension classification model. + +3. **No new cap-level mechanism is needed for multi-instance.** The instance number + (from the control layer, see §13) is sufficient. The cap code shape is the same; + only the call-site indexing expression differs based on the declared dimension roles. + +--- + +## 13. Cross-cutting design decision: how host data enters the cap chain + +Across all four models, two mechanisms are used for getting host model data into the +generated caps: + +| Mechanism | Models using it | Description | +|-----------|----------------|-------------| +| **Module USE** | UFS, SCM, CAM-SIMA, NEPTUNE | Static API has `use ccpp_data, only: gfs_statein, ...`. Data module name is known at generation time. | +| **Command-line arguments** | capgen (optional) | Generator accepts host variable access paths as CLI flags; generated caps receive data as explicit dummy arguments. | + +### 13.1 The capgen dual-mechanism problem + +Capgen supports both mechanisms, and this is a direct source of its complexity. The +variable-matching logic, VarDictionary scope chains, and `CCPPDatabaseObj` all exist +partly to handle the routing of variables that may arrive via either path. Maintaining +two entry points to the data layer doubles the surface area that must be tested and +reasoned about. + +### 13.2 The proposed single-mechanism approach + +The redesign will use **module USE exclusively** for all host data. The reasoning: + +- All four production models already use module USE, including CAM-SIMA (the capgen + model), which does not use capgen's CLI-argument path in practice. +- Module names are stable, known at generation time, and make the generated code + self-documenting (`use ccpp_data, only: gfs_statein` is unambiguous). +- Eliminating the CLI-argument entry path eliminates an entire class of generator + complexity. + +### 13.3 Runtime control variables — the thin explicit layer + +While all *data* enters via module USE, a set of *control* variables must be passed at +runtime because they change from call to call. These are not physics data; they tell the +cap *how* to index into the data it already has access to: + +| Variable | Purpose | When it matters | +|----------|---------|----------------| +| `ccpp_instance` | Select the instance dimension in host variables | NEPTUNE (N>1); others use 1 | +| `ccpp_thread_no` | Index optional pointer arrays per thread | Run phase with OpenMP | +| `horizontal_loop_begin` | Start of horizontal chunk to process | Run phase | +| `horizontal_loop_end` | End of horizontal chunk to process | Run phase | +| `ccpp_nthreads` | Max threads available for internal physics use | Non-run phases (currently `gfs_control%nthreads`) | +| `errmsg` / `errflg` | Error reporting return path | All phases | + +These are exactly the values that `cdata` carries in the current implementation. +Whether they are packaged as a `ccpp_t` struct or passed as individual named arguments to +`ccpp_physics_*` is an open design question for implementation. Either way, the generator +only needs to know about these variables and their standard names — it does not need to +accept host data paths on the command line. + +### 13.4 The dimension classification model + +A host variable's metadata declares the **standard name of each of its dimensions** in +order. The generator classifies every dimension into one of three categories and +constructs the call-site expression accordingly. + +**Category 1 — Registered dimensions.** The generator knows the semantics of these +standard names and generates special call-site expressions for them: + +| Standard name | Call-site expression | Notes | +|--------------|---------------------|-------| +| `instance_dimension` | `instance_number` (scalar index) | Omitted if variable has no instance dimension | +| `horizontal_dimension` | `1:horizontal_dimension` (non-run) or `horiz_begin:horiz_end` (run) | Phase-dependent | +| `vertical_dimension` | `1:vertical_dimension` | Fixed range | + +`instance_dimension` has the same registered status as `horizontal_dimension` and +`vertical_dimension`. Single-instance models simply do not declare any variables with +an `instance_dimension`, and the generator omits that index entirely. + +**Category 2 — Arbitrary host-declared dimensions.** Any dimension whose standard name +is not in the registered set. These are declared in host metadata pointing to a Fortran +expression accessible via module USE — either a flat module variable or a DDT member +(e.g. `gfs_control%ntrac`, `gfs_control%kice`). The generator emits `1:expression` +at the call site, resolved at generation time from the metadata. Fixed-index extractions +(e.g. `gfs_statein%qgrs(..., gfs_control%ntqv)`) are a special case: the dimension +value is a scalar index rather than a range upper bound, and the metadata must declare +which case applies. + +**Category 3 — Optional selector.** Not a dimension per se, but a boolean `active` +condition declared in variable metadata. Generates a pointer-association guard around +the call site (the pattern described in §9 and §11). + +This three-category model works uniformly regardless of host layout: +- `gfs_statein(instance)%tair(horiz, vert)` — registered instance + registered horizontal + registered vertical +- `flat_field(instance, horiz, vert, ntrac)` — registered + registered + registered + arbitrary +- `flat_field(horiz, vert)` — no instance dimension, single-instance model + +No special-casing per host model is needed in the generator. + +### 13.5 `type = control` — metadata declaration for runtime control variables + +The registered dimensions (§13.4 Category 1) are *dimension names* that appear in a +variable's `dimensions = (...)` list. Their actual *runtime values* are supplied by a +separate set of variables declared with `type = control` in host metadata. + +| `type = control` standard name | Fills in registered dimension / purpose | +|-------------------------------|----------------------------------------| +| `ccpp_instance` | `instance_dimension` — scalar index selecting the active instance | +| `ccpp_thread_no` | Not a dimension; indexes optional pointer arrays per thread | +| `horizontal_loop_begin` | Lower bound of `horizontal_dimension` in run phase | +| `horizontal_loop_end` | Upper bound of `horizontal_dimension` in run phase | +| `ccpp_nthreads` | Not a dimension; max threads available for internal physics use | +| `errmsg` / `errflg` | Error reporting return path | + +Variables declared `type = control` are: +- **Passed explicitly as runtime arguments** to `ccpp_physics_*` by the host driver + (not accessed via module USE, because their values change per call) +- **Used by the generator** to construct call-site indexing expressions for registered + dimensions, and to generate the `ccpp_nthreads` assignment before non-run scheme calls +- **Available to physics schemes** by standard name like any other variable — if a scheme + declares a variable with a matching standard name (e.g. `ccpp_nthreads`, + `horizontal_loop_begin`), the framework passes it as a scheme argument in the normal way + +This is similar in concept to capgen's `type = host` annotation but with a narrower, +well-defined scope. The name `control` is intentional: these variables *control* how +the cap indexes into the data, not what the data is. + +The set of recognized standard names for `type = control` variables is fixed and small. +Declaring them explicitly in metadata — rather than having the generator recognize magic +names — keeps the mechanism open and self-documenting. + +### 13.6 Consequences for the generator + +1. The generator reads host metadata to learn: + - Module names for all host data variables (emitted as `use` statements in the static API) + - The dimension standard names of each variable (for call-site expression construction) + - Which variables are `type = control` (for the runtime argument layer) +2. At cap generation time, the static API's `use` statements are emitted from the module + names — no runtime flexibility, no CLI data routing. +3. Call-site subsetting for every variable is constructed purely from its declared + dimension standard names: registered dimensions use the Category 1 rules; arbitrary + dimensions are resolved to Fortran expressions via the host metadata. +4. The only runtime inputs to the cap are the `type = control` variables. Their values + are supplied by the host driver for each `ccpp_physics_*` call. diff --git a/doc/redesign_analysis_original_202060505T2044.md b/doc/redesign_analysis_original_202060505T2044.md new file mode 100644 index 00000000..2248f1c0 --- /dev/null +++ b/doc/redesign_analysis_original_202060505T2044.md @@ -0,0 +1,2489 @@ +# CCPP Framework Code Generator — Technical Analysis for Redesign + +*Analysis date: 2026-05-04. Clarifications added: 2026-05-05.* + +This document is a deep-dive technical analysis of the two existing CCPP Framework code generators — +`ccpp-prebuild` and `ccpp-capgen` — produced as input to a planned complete redesign. +It covers execution flow, data structures, feature sets, build system integration, and +key architectural differences. + +--- + +## Table of Contents + +1. [Background and motivation](#1-background-and-motivation) +2. [ccpp-prebuild — detailed analysis](#2-ccpp-prebuild--detailed-analysis) +3. [ccpp-capgen — detailed analysis](#3-ccpp-capgen--detailed-analysis) +4. [Shared infrastructure](#4-shared-infrastructure) +5. [Feature comparison](#5-feature-comparison) +6. [Build system integration](#6-build-system-integration) +7. [Key architectural differences](#7-key-architectural-differences) +8. [Design considerations for the redesign](#8-design-considerations-for-the-redesign) +9. [Real-world example: CCPP Single Column Model (SCM)](#9-real-world-example-ccpp-single-column-model-scm) +10. [Real-world example: CAM-SIMA (capgen)](#10-real-world-example-cam-sima-capgen) +11. [Real-world example: UFS Weather Model (prebuild)](#11-real-world-example-ufs-weather-model-prebuild) +12. [Real-world example: Navy NEPTUNE (prebuild, restricted)](#12-real-world-example-navy-neptune-prebuild-restricted) +13. [Cross-cutting design decision: how host data enters the cap chain](#13-cross-cutting-design-decision-how-host-data-enters-the-cap-chain) + +--- + +## 1. Background and motivation + +The CCPP Framework is a code generator that analyzes metadata describing variables required +by physical parameterizations in numerical weather prediction (NWP) models, compares them +against metadata provided by a host model, and generates Fortran interface ("cap") code that +connects the two. + +There are two generations of the generator: + +**`ccpp-prebuild`** (`scripts/ccpp_prebuild.py`): +- Simple, mostly procedural Python +- Used in: NOAA UFS Weather Model, Navy NEPTUNE, CCPP-SCM +- Extremely reliable in research, development, and operations +- Fewer capabilities; simpler design + +**`ccpp-capgen`** (`scripts/ccpp_capgen.py`): +- Highly complex, object-oriented Python taken to the extreme +- Used in: NCAR CAM-SIMA (still mostly a research/development model) +- Many advanced features designed but never implemented (funding/priority gaps) +- Notoriously difficult to develop; no remaining team member fully understands it + +**The original plan** was to update `ccpp-capgen` with missing features from `ccpp-prebuild` +and transition all models to it. **This plan has been abandoned** in favor of a complete +redesign that draws the best lessons from both generations. + +The immediate trigger for abandoning capgen was the failure — after considerable effort by +three developers — to make capgen pass DDT arguments to group caps the way prebuild does. +This is the root cause of capgen's severe performance problem (seconds for prebuild, +10+ minutes for capgen on the same suite set) and of its broken handling of optional +variables under Fortran compiler debugging flags. + +--- + +## 2. ccpp-prebuild — detailed analysis + +### 2.1 Command-line arguments and configuration + +Entry point: `scripts/ccpp_prebuild.py`, `main()`. + +Arguments parsed by `argparse`: + +| Argument | Required | Purpose | +|---|---|---| +| `--config` | yes | Path to host-model Python config module | +| `--suites` | no | Comma-separated suite names (without `.xml`) | +| `--builddir` | no | Override build directory from config | +| `--namespace` | no | Appended to static API module name | +| `--debug` | no | Insert Fortran array-size checks in generated caps | +| `--clean` | no | Remove generated files and exit | +| `--verbose` | no | Set logging to DEBUG | + +The `--config` file is a plain Python module imported dynamically via `importlib`. +Key variables it must define: + +| Config variable | Purpose | +|---|---| +| `VARIABLE_DEFINITION_FILES` | List of host-model Fortran sources with metadata hooks | +| `SCHEME_FILES` | List of physics scheme Fortran sources | +| `CAPS_DIR` | Output directory for generated cap `.F90` files | +| `SUITES_DIR` | Directory containing suite definition XML files | +| `STATIC_API_DIR` | Output directory for `ccpp_static_api.F90` | +| `TYPEDEFS_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for typedef build snippets | +| `SCHEMES_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for scheme build snippets | +| `CAPS_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for cap build snippets | +| `HTML_VARTABLE_FILE`, `LATEX_VARTABLE_FILE` | Documentation output paths | +| `TYPEDEFS_NEW_METADATA` | Optional: dict enabling DDT member name translation bridge | + +The config file can contain arbitrary Python expressions — computed file lists, +conditional logic, environment-variable lookups — making it very flexible. + +### 2.2 Step-by-step execution pipeline + +``` +1. Import config module dynamically via importlib + +2. gather_variable_definitions() + for each file in VARIABLE_DEFINITION_FILES: + parse_variable_tables(file) [metadata_parser.py] + → metadata_define: OrderedDict[standard_name → [mkcap.Var]] + +3. collect_physics_subroutines() + for each file in SCHEME_FILES: + parse_scheme_tables(file) [metadata_parser.py] + → metadata_request: OrderedDict[standard_name → [mkcap.Var, ...]] + → arguments_request: OrderedDict[scheme → OrderedDict[subroutine → [std_names]]] + → dependencies_request: OrderedDict[scheme → [abs_paths]] + → schemes_in_files: OrderedDict[scheme → abs_path] + +4. compare_metadata() [batch matching] + for each std_name in metadata_request: + check exists in metadata_define + check type/kind/rank compatibility + register unit conversions in var.actions + copy local_name as var.target + → metadata: OrderedDict[std_name → [Var]] (targets and actions set) + +5. check_optional_arguments() [warnings only] + +6. For each requested suite XML: + Suite.parse(xml) [mkstatic.py] → Suite + Group objects + Group.write() → ccpp___cap.F90 + Suite.write() → ccpp__cap.F90 + +7. API.write() [mkstatic.py] + → ccpp_static_api[_].F90 + +8. Write build-system snippets [mkcap.py writers] + → CCPP_CAPS.cmake/mk/sh + → CCPP_SCHEMES.cmake/mk/sh + → CCPP_TYPEDEFS.cmake/mk/sh + → CCPP_API.cmake/sh + +9. mkdoc.metadata_to_html() → HTML variable table + mkdoc.metadata_to_latex() → LaTeX variable table +``` + +### 2.3 Data structures — the "flat dict" model + +Everything in prebuild lives in flat Python `OrderedDict` structures. There is no object +hierarchy; variables are simple Python objects with plain attributes. + +```python +# Top-level data containers +metadata_define: OrderedDict[standard_name → [mkcap.Var]] # 1 Var per std_name +metadata_request: OrderedDict[standard_name → [mkcap.Var, ...]] # N Vars (one per scheme×subroutine) +arguments_request: OrderedDict[scheme_name → OrderedDict[subroutine_name → [std_names]]] +dependencies_request: OrderedDict[scheme_name → [abs_paths]] +schemes_in_files: OrderedDict[scheme_name → abs_path] +``` + +`mkcap.Var` attributes: + +| Attribute | Type | Description | +|---|---|---| +| `standard_name` | str | CF-convention unique identifier | +| `long_name` | str | Human-readable description | +| `units` | str | Physical units | +| `local_name` | str | Fortran local name (may be DDT member reference) | +| `type` | str | Fortran type (real, integer, logical, or DDT name) | +| `kind` | str | Fortran kind parameter | +| `dimensions` | list[str] | Dimension standard names | +| `intent` | str | in / out / inout | +| `active` | str | `'T'`, `'F'`, or expression string | +| `optional` | str | `'T'` or `'F'` | +| `pointer` | bool | Whether Fortran POINTER attribute needed | +| `target` | str | Set during matching: the host model local_name | +| `actions` | dict | `{'in': fn, 'out': fn}` for unit conversions | +| `container` | str | Encoded provenance: `MODULE_foo SCHEME_bar SUBROUTINE_baz` | + +**Performance note on `container` and `target`**: these two attributes act as a lookup +cache computed once during the `compare_metadata()` batch step. The `container` string +encodes where each variable lives in the host model (module and, if applicable, the +DDT member chain). The `target` records the resolved Fortran local name. Both are +computed once and then used directly during Fortran cap generation — no further dictionary +lookups are needed. This is a major contributor to prebuild's speed advantage. + +### 2.4 Metadata parsing and the bridge to capgen + +`metadata_parser.py` is a shared module that acts as a bridge. It detects whether a +metadata section in a Fortran source file uses the old pipe-delimited format (deprecated, +warning emitted) or the new `.meta` format (triggered by `!! \htmlinclude .html` +in the Fortran source comment hook). + +For `.meta` files, `read_new_metadata()` in `metadata_parser.py`: +1. Calls capgen's `metadata_table.parse_metadata_file()` → `[MetadataTable]` +2. Converts each `metavar.Var` to a `mkcap.Var` +3. Normalizes `active` to `'T'`/`'F'`/expression, `optional` to `'T'`/`'F'` + +The `TYPEDEFS_NEW_METADATA` config variable (when provided) triggers an additional +pass via `convert_local_name_from_new_metadata()` which translates flat +standard-name-style local names into DDT member references such as +`Atm(blk_no)%q(:,:,:,graupel_index)`. This is the bridge that makes the newer +`.meta` format work with the older DDT-heavy host model code. + +### 2.5 Variable matching — `compare_metadata()` + +A single batch function processes all matching. For each standard name in `metadata_request`: + +1. Check it exists in `metadata_define` — error if missing +2. Check there is exactly one definition — error if ambiguous +3. Call `var.compatible(other_var)` — checks equality of `standard_name`, `type`, `kind`, and rank +4. Register unit conversions: if units differ, `var.convert_from()` / `var.convert_to()` + stores a conversion function in `var.actions` +5. Check `active` attribute: if host variable is conditionally allocated and scheme variable + is not `optional`, issue a warning (not an error) +6. Copy `local_name` from the define side as `var.target` +7. Build module use list from container strings + +Result: `metadata` dict where each `Var` has `.target` set to the host model local name +and `.actions` populated with any needed unit conversion functions. + +### 2.6 Generated Fortran files + +#### Group cap: `ccpp___cap.F90` + +One module per group. For each CCPP stage (tsinit, init, run, tsfinal, finalize), a subroutine: + +```fortran +module ccpp_suite_A_physics_cap + use scheme_module, only: scheme_run + use host_module_A, only: ddt_A ! DDT, not flat fields + use host_module_B, only: ddt_B + implicit none + contains + + subroutine suite_A_physics_run_cap(ddt_A, ddt_B, im, iaend, ierr, ...) + type(ddt_A_type), intent(inout), target :: ddt_A ! entire DDT passed + type(ddt_B_type), intent(inout), target :: ddt_B + integer, intent(in) :: im, iaend ! loop bounds + integer, intent(out) :: ierr + logical, save :: initialized(200) = .false. + ! optional variable: local pointer, conditionally associated + real(kind_phys), pointer :: opt_var(:) => null() + if (ddt_A%active_flag) then + opt_var => ddt_A%opt_field + end if + ! unit conversion: local variable + real(kind_phys) :: converted_var(im) + converted_var(:) = ddt_B%field(:im) * conversion_factor + ! fixed-index extraction: local pointer for a specific tracer + real(kind_phys), pointer :: q_water_vapor(:,:) => null() + q_water_vapor => ddt_A%q(:,:,ntqv) ! ntqv = water vapor index in tracer array + ! call scheme with loop-bound application and extracted variables at the call site + call scheme_run( & + arg1 = ddt_A%field1(1:im), & ! horizontal loop-bound applied here + arg2 = ddt_A%field2(1:im,:), & ! loop-bound + all levels + qv = q_water_vapor(1:im,:), & ! specific tracer, loop-bound applied + arg3 = converted_var, & ! unit-converted local var + opt_arg = opt_var, & ! optional pointer + ...) + if (ierr /= 0) return + end subroutine +end module +``` + +Key points: +- **DDTs are passed as arguments, not flat fields.** Hundreds of variables arrive as + one or a small number of DDT arguments. This is the fundamental architectural choice + that makes prebuild fast and safe with compiler debugging flags. +- **Two distinct "subsetting" operations happen at the scheme call site:** + 1. *Loop-bound application*: horizontal range `1:im` (or `im` for scalar extents) + applied in the scheme call argument expressions. + 2. *Fixed-index extraction*: a specific element along one dimension is selected, + e.g. `q_water_vapor => ddt%q(:,:,ntqv)` extracts the water vapor tracer from the + full tracer array. A local pointer (or local variable for unit conversions) is + declared just before the scheme call and passed as the scheme argument. The group + cap always receives the full data; these extractions are local to the group cap. +- **Optional variables** are handled by declaring a local `pointer` variable and + conditionally associating it with the DDT field based on the `active` expression. + An unassociated pointer is passed to the scheme if the variable is inactive. This + avoids compiler exceptions when mandatory debugging flags are enabled, because the + unallocated field is never directly referenced — only the already-null pointer is. +- `logical :: initialized(200), save` — per-instance initialization tracking. The + 200 is the maximum number of complete model instances that can coexist in memory + simultaneously (used in ensemble approaches where multiple copies of the full model + state live in memory at once). Each instance has its own initialization flag. +- For the `run` phase, `im` and `iaend` (or similar) carry `horizontal_loop_begin` + and `horizontal_loop_end`, enabling OpenMP thread-level parallelism where each + thread processes a horizontal slice. +- Explicit keyword argument passing in scheme calls. +- Unit conversion: a local variable is declared and populated before the call; the + local variable is then passed to the scheme. +- Error check after each scheme call; returns immediately on error. +- `--debug` flag inserts Fortran array-size assertions. + +#### Suite cap: `ccpp__cap.F90` + +Imports all group cap functions and exposes one function per stage that chains group calls. + +#### Static API: `ccpp_static_api[_].F90` + +A single Fortran module `ccpp_static_api` with one subroutine per stage: + +```fortran +subroutine ccpp_physics_run(cdata, suite_name, group_name, ierr) + character(len=*), intent(in) :: suite_name, group_name + select case(trim(suite_name)) + case('suite_A') + select case(trim(group_name)) + case('physics') + call suite_A_physics_run_cap(cdata, ierr) + ... + end select + ... + end select +end subroutine +``` + +This is the **single entry point** the host model calls. The host model passes `suite_name` +and `group_name` at runtime; the static API dispatches to the appropriate cap function. + +### 2.7 Build system snippet files generated + +Six output files (Makefile, CMakefile, shell source) for three variable sets: + +| File | Content | +|---|---| +| `CCPP_CAPS.cmake` | `set(CAPS /abs/path/cap1.F90 /abs/path/cap2.F90 ...)` | +| `CCPP_SCHEMES.cmake` | `set(SCHEMES /abs/path/scheme1.F90 ...)` | +| `CCPP_TYPEDEFS.cmake` | `set(TYPEDEFS module1 module2 ...)` (module names, not paths) | +| `CCPP_API.cmake` | `set(API /abs/path/ccpp_static_api.F90)` | + +All files are written as `.tmp` first and compared against the existing version; they are +replaced only if the content changed, which avoids unnecessary recompilation of downstream +Fortran targets. + +### 2.8 What `mkcap.py`, `mkstatic.py`, and `mkdoc.py` each do + +**`mkcap.py`**: +- Defines the `mkcap.Var` class (prebuild's variable data class) +- Defines six file-writer classes: `CapsMakefile`, `CapsCMakefile`, `CapsSourcefile`, + `SchemesMakefile`, `SchemesCMakefile`, `SchemesSourcefile`, `TypedefsMakefile`, + `TypedefsCMakefile`, `TypedefsSourcefile` +- Each writer has a `write(file_list)` method that produces a formatted include file +- Does NOT generate any Fortran + +**`mkstatic.py`**: +- Defines `Suite`, `Group`, `Subcycle` classes that parse suite definition XML and + generate Fortran caps +- `Suite.parse()`: reads SDF XML via `xml.etree.ElementTree`, builds `Group` and + `Subcycle` objects +- `Suite.write()`: drives cap generation for all groups and the suite-level cap +- `Group.write()`: generates the group cap Fortran — argument list construction, + module `use` statements, unit conversion code, scheme calls, error handling +- Defines `API` class: generates the static API Fortran module (suite_name/group_name + dispatch switch) +- `CCPP_SUITE_VARIABLES` dict: mandatory variables always included (error message, + error code, loop counter, loop extent) +- Helper functions `extract_parents_and_indices_from_local_name()` and + `extract_dimensions_from_local_name()` handle complex DDT member access like + `Atm(blk_no)%q(:,:,:,graupel_index)` — these are critical for DDT-heavy host models + +**`mkdoc.py`**: +- `metadata_to_html()`: produces an HTML table of all host-model provided variables + (standard_name, long_name, units, rank, type, kind, source, local_name) +- `metadata_to_latex()`: produces a LaTeX table combining host-defined and scheme-requested + variables, annotating which schemes use each variable and whether unit conversion is needed +- Informational outputs only; do not affect the build + +--- + +## 3. ccpp-capgen — detailed analysis + +### 3.1 Command-line arguments + +Entry point: `scripts/ccpp_capgen.py`, `_main_func()`. +Arguments parsed via `framework_env.parse_command_line()` into a `CCPPFrameworkEnv` object: + +| Argument | Required | Purpose | +|---|---|---| +| `--host-files` | yes | `.meta` files or `.txt` indirect file lists | +| `--scheme-files` | yes | Same format | +| `--suites` | yes | `.xml` SDF files or `.txt` lists | +| `--output-root` | no | Directory for generated files | +| `--host-name` | no | If given, generates a host cap | +| `--ccpp-datafile` | no | Path for datatable XML (default: `datatable.xml`) | +| `--kind-type` | no (repeatable) | Fortran kind mappings, e.g. `kind_phys=REAL64` | +| `--preproc-directives` | no | Fortran preprocessor macros | +| `--use-error-obj` | no | Use error object instead of scalar error variables | +| `--force-overwrite` | no | Always regenerate output | +| `--clean` | no | Remove files listed in datatable and exit | +| `--verbose` | no (repeatable) | Increase log verbosity | + +`CCPPFrameworkEnv` (defined in `framework_env.py`) consolidates all settings into typed +properties and stores a `kind_dict` mapping CCPP kind names to `[kind_spec, module]` pairs. + +### 3.2 Step-by-step execution pipeline + +``` +1. create_file_list() + expand .txt indirect file lists, validate .meta extensions + +2. register_ddts(scheme_files) + pre-scan all scheme .meta files + register DDT type names via register_fortran_ddt_name() + (so the host parser can recognize them as non-intrinsic types) + +3. parse_host_model_files() + for each host .meta file: + metadata_table.parse_metadata_file() → [MetadataTable] + find_associated_fortran_file() → matching .F90 path + parse_fortran_file() → Fortran declarations (via fortran_tools) + check_fortran_against_metadata() → cross-validation (type, kind, rank, intent) + accumulate MetadataSection headers: DDT, module, host types + +4. HostModel(table_dict, host_name, run_env) + process DDT headers: → DDTLibrary + ddt_dict (VarDictionary) + process module/host headers: → main VarDictionary + __var_locations + add ConstituentVarDict synthetically for ccpp_model_constituents_t + +5. API(sdfs, host_model, scheme_headers, run_env) + for each SDF XML: + Suite construction: + auto-create 5 phase groups: register, initialize, timestep_initial, + timestep_final, finalize + parse elements → Group objects (RUN_PHASE_NAME) + parse / tags → Scheme objects in full-phase groups + Suite.analyze(host_model, scheme_library, ddt_library, run_env): + Group.analyze() → Scheme.analyze(): + for each scheme argument: + VarDictionary.find_variable() [scope chain search] + Var.compatible() [→ VarCompatObj with transformations] + loop dim substitution for _run phase + register constituent if constituent=True + variable promotion: group outputs → suite level if needed by later group + +6. ccpp_api.write(outdir, run_env) + suite cap .F90 per suite + group caps (embedded or separate) + host cap .F90 (if --host-name given) + ccpp_kinds.F90 + +7. generate_ccpp_datatable() → datatable.xml +``` + +### 3.3 Object hierarchy + +``` +API (ccpp_suite.py) + └── Suite (extends VarDictionary) [one per SDF XML] + parent → ConstituentVarDict (extends VarDictionary) + parent → API + ├── Group (suite_objects.py, extends VarDictionary) [one per ] + │ call_list: CallList (extends VarDictionary) + │ ├── Subcycle (suite_objects.py) + │ │ └── Scheme (suite_objects.py, extends SuiteObject) + │ └── Scheme (for full-phase groups: init, register, etc.) + └── (auto groups: register, initialize, timestep_initial, + timestep_final, finalize) + +HostModel (host_model.py, extends VarDictionary) + ├── ddt_lib: DDTLibrary + │ └── {ddt_name → MetadataSection} + ├── ddt_dict: VarDictionary (all DDT field variables, expanded) + └── loop_vars: VarDictionary (run-time dimension variables) + +VarDictionary (metavar.py) + ├── {standard_name → Var} + └── parent_dict → VarDictionary ← scope chain for find_variable() + +Var (metavar.py) + └── __prop_dict: {property_name → validated_value} + +VarDDT (ddt_library.py, extends Var) + └── __field: Var | VarDDT ← recursive DDT traversal chain +``` + +### 3.4 Variable matching — scope-chain and VarCompatObj + +Unlike prebuild's single batch `compare_metadata()`, capgen performs incremental, +scope-aware matching during the suite analysis phase. + +For each scheme argument in `Scheme.analyze()`: +1. `VarDictionary.find_variable(standard_name)` — searches scope chain: + local group dict → suite dict → ConstituentVarDict → host model dict +2. `Var.compatible(other, run_env)` returns a `VarCompatObj` — not a bool. + `VarCompatObj` carries: + - Whether the variables are equivalent (no transformation needed) + - Whether they are compatible with transformations (unit conversion, dimension + substitution, `top_at_one` flip) + - The reason for any incompatibility (for error messages) +3. For `_run` phase: `horizontal_dimension` is automatically substituted with + `horizontal_loop_begin:horizontal_loop_end` +4. For `constituent = True` variables: auto-registered in `ConstituentVarDict`; + allocation/management code is generated +5. Variable promotion: if a Group produces a variable needed by a later Group, it is + promoted to Suite-level scope + +`VarCompatObj` compatibility considers: +- Type equality +- Kind equality (with ISO kind aliases) +- Units compatibility (triggers unit conversion if compatible) +- Dimension substitutability (horizontal loop vs. full dimension, vertical extent) +- `top_at_one` orientation (triggers flip if needed) +- `protected` status (cannot be an output if protected) +- `CCPP_HORIZONTAL_DIMENSIONS`, `CCPP_VERTICAL_DIMENSIONS`, `CCPP_LOOP_DIM_SUBSTS` + from `var_props.py` + +### 3.5 `metavar.Var` properties + +`metavar.Var` stores all properties in a validated `__prop_dict`. Properties: + +**Specification properties** (all metadata contexts): + +| Property | Type | Notes | +|---|---|---| +| `local_name` | str | Valid Fortran identifier | +| `standard_name` | str | CF-convention, lowercase+underscores | +| `long_name` | str | Human-readable description | +| `units` | str | Physical units string | +| `dimensions` | list | Dimension standard names or `()` | +| `type` | str | Intrinsic or registered DDT name | +| `kind` | str | Fortran kind parameter | +| `active` | str | Conditional allocation expression | +| `optional` | bool | Whether scheme can handle missing var | +| `protected` | bool | Cannot be written by schemes | +| `allocatable` | bool | Has ALLOCATABLE attribute | +| `state_variable` | bool | Persists across timesteps | +| `persistence` | str | `timestep` or `run` | +| `default_value` | str | Fortran expression | +| `diagnostic_name` | str | Diagnostic output name | +| `target` | bool | Has TARGET attribute | +| `polymorphic` | bool | CLASS(*) type | +| `top_at_one` | bool | Vertical ordering: top at index 1 | + +**Scheme-only properties**: + +| Property | Type | Notes | +|---|---|---| +| `intent` | str | in / out / inout | + +**Constituent properties**: + +| Property | Type | Notes | +|---|---|---| +| `constituent` | bool | Is a CCPP-managed constituent (tracer) | +| `advected` | bool | Is advected by the dynamical core | +| `molar_mass` | float | Molecular weight (positive) | + +### 3.6 Capgen-only features + +**Fortran cross-validation** (`check_fortran_against_metadata()`): +- Parses the actual `.F90` file alongside the `.meta` file +- Checks that every metadata entry matches the real Fortran declaration: + variable count, local_name, type, kind, intent (for schemes), dimension rank/names +- Catches bugs where metadata was updated but the Fortran source was not (or vice versa) + +**State machine** (`ccpp_state_machine.py`, `state_machine.py`): +- `CCPP_STATE_MACH`: a `StateMachine` instance with 6 transitions +- Valid state sequence: `register → uninitialized → initialized → in_time_step` +- Suite caps include a `character(len=16) :: ccpp_suite_state` variable +- State-checking code at the start of each phase function enforces correct call ordering +- `CCPP_STATE_MACH.function_match()` uses compiled regex to identify which CCPP phase + a subroutine name belongs to + +**Constituent variable support** (`constituents.py`): +- `ConstituentVarDict` (extends `VarDictionary`) manages traceable species (tracers) +- When a scheme declares `constituent = True`, `find_variable()` auto-creates the variable +- Allocation code for the constituent array is auto-generated +- Constants: `CONST_DDT_NAME = "ccpp_model_constituents_t"`, + `CONST_PROP_TYPE = "ccpp_constituent_properties_t"` + +**DDT library** (`ddt_library.py`): +- `VarDDT(Var)`: represents a DDT field variable at any nesting level +- Traversal chain: `VarDDT → VarDDT → ... → Var` (innermost is the actual leaf field) +- `DDTLibrary`: dictionary of DDT `MetadataSection` objects +- `collect_ddt_fields()` expands DDT variables into component fields in `ddt_dict` + +**Host cap generation** (`host_cap.py`): +- Generated only when `--host-name` is given +- Produces `_ccpp_cap.F90` +- Subroutines: `_ccpp_physics_(api_vars)` + that call into suite cap functions + +**`ccpp_kinds.F90`**: +- Simple Fortran module `ccpp_kinds` containing `use` statements that import all kind + parameters specified via `--kind-type` +- Makes kind parameters available to both schemes and caps without circular dependencies + +**Datatable XML** (`ccpp_datafile.py`): +- Produced after generation; lists all generated files, scheme entries, variable properties, + suite configurations +- Queryable by the build system via `ccpp_datafile.py --suite-files` etc. +- Supports `--clean` workflow: reads the file list, removes all generated files, deletes itself +- `DatatableReport` class provides a programmatic query API + +**In-memory database** (`ccpp_database_obj.py`): +- `CCPPDatabaseObj`: wraps `HostModel` and `API` for programmatic access to capgen results +- Returned when `capgen()` is called with `return_db=True` +- Provides `host_model_dict()`, `suite_list()`, `constituent_dictionary(suite)` + +**Variable tracking tool** (`ccpp_track_variables.py`): +- Standalone diagnostic: traces a specific variable through a suite, showing which schemes + use it and with what intent +- Uses prebuild's `import_config` and capgen's `Suite`/`parse_metadata_file` together + +**Fortran-to-metadata tool** (`ccpp_fortran_to_metadata.py`): +- Standalone utility: parses annotated Fortran source files and generates skeleton `.meta` + files — used to bootstrap new scheme metadata + +--- + +## 4. Shared infrastructure + +### 4.1 Module sharing map + +| Module | Used by prebuild | Used by capgen | Notes | +|---|---|---|---| +| `metadata_parser.py` | yes | partial | **Bridge module**: calls capgen's parser, returns mkcap.Var | +| `metadata_table.py` | via bridge | yes (primary) | Native `.meta` format parser | +| `metavar.py` | no | yes | Primary `Var` class, `VarDictionary` | +| `var_props.py` | no | yes | `VariableProperty`, `VarCompatObj`, dimension constants | +| `mkcap.py` | yes | no | `mkcap.Var` class + build-snippet writers | +| `mkstatic.py` | yes | no | Suite/Group/API Fortran generators | +| `mkdoc.py` | yes | no | HTML/LaTeX documentation generators | +| `common.py` | yes | partial | `CCPP_STAGES`, container encoding | +| `framework_env.py` | dummy instance | yes (primary) | `CCPPFrameworkEnv` | +| `file_utils.py` | no | yes | `create_file_list`, `move_modified_files` | +| `code_block.py` | no | yes | Structured Fortran output | +| `ddt_library.py` | no | yes | `DDTLibrary`, `VarDDT` | +| `host_model.py` | no | yes | `HostModel` class | +| `host_cap.py` | no | yes | Host cap generation | +| `ccpp_suite.py` | no | yes | `Suite`, `API` classes | +| `suite_objects.py` | no | yes | `Scheme`, `Group`, `Subcycle`, `CallList` | +| `constituents.py` | no | yes | `ConstituentVarDict` | +| `ccpp_datafile.py` | no | yes | Datatable XML | +| `ccpp_database_obj.py` | no | yes | `CCPPDatabaseObj` | +| `ccpp_state_machine.py` | no | yes | `CCPP_STATE_MACH` | +| `state_machine.py` | no | yes | `StateMachine` base class | +| `ccpp_fortran_to_metadata.py` | no | yes | Fortran→metadata bootstrap tool | +| `ccpp_track_variables.py` | partial | partial | Uses both worlds | + +**The key architectural debt**: `metadata_parser.py` is a prebuild module that internally +calls capgen's `metadata_table.parse_metadata_file()` and converts results to `mkcap.Var` +objects. This creates a one-way dependency (prebuild → capgen's parser infrastructure) +while presenting a prebuild-style API to `ccpp_prebuild.py`. It exists only because +prebuild predates the `.meta` format. + +### 4.2 The `.meta` file format + +The `.meta` format is the native format for capgen and the expected format for all new +scheme development. The Fortran source file contains a comment hook pointing to the `.meta` +file: + +```fortran +!! \section arg_table_scheme_name_run Argument Table +!! \htmlinclude scheme_name_run.html +``` + +The `.meta` file itself uses an INI-style format: + +```ini +[ccpp-table-properties] + name = scheme_name + type = scheme + dependencies_path = ../some/path + dependencies = utility_module.F90, another.F90 + +[ccpp-arg-table] + name = scheme_name_run + type = scheme +[ im ] + standard_name = horizontal_loop_extent + long_name = horizontal loop extent + units = count + type = integer + dimensions = () + intent = in +[ dz ] + standard_name = layer_thickness + long_name = thickness of each model layer + units = m + type = real + kind = kind_phys + dimensions = (horizontal_loop_extent, vertical_layer_dimension) + intent = in +``` + +Multiple `[ccpp-arg-table]` sections are allowed in a scheme file (one per phase: +`_init`, `_run`, `_finalize`, `_timestep_init`, `_timestep_finalize`). +Singleton tables (DDT, module, host) allow only one section. + +### 4.3 Variable property validation (`var_props.py`) + +`VariableProperty` encapsulates a single metadata property with its name, Python type, +optionality, default, valid-value constraints, and a check function. Check functions used: + +| Checker | What it validates | +|---|---| +| `check_local_name` | Valid Fortran identifier | +| `check_cf_standard_name` | Lowercase, underscores, alphanumeric only | +| `check_fortran_type` | Intrinsic type or registered DDT name | +| `check_units` | Valid unit string (normalizes `+` in exponents) | +| `check_dimensions` | Valid dimension specification | +| `check_default_value` | Valid Fortran expression | +| `check_molar_mass` | Positive float (for constituents) | + +`CCPP_HORIZONTAL_DIMENSIONS`, `CCPP_VERTICAL_DIMENSIONS`, `CCPP_LOOP_DIM_SUBSTS` +in `var_props.py` define the recognized dimension forms and the run-time substitution +map (e.g., `horizontal_dimension → horizontal_loop_begin:horizontal_loop_end`). + +--- + +## 5. Feature comparison + +| Feature | prebuild | capgen | Notes | +|---|---|---|---| +| **Input formats** | | | | +| Native `.meta` format | via bridge | yes | | +| Old pipe-delimited format | deprecated warn | not supported | | +| **Parsing and validation** | | | | +| Fortran source cross-validation | no | yes | capgen parses actual .F90 to cross-check | +| Preprocessor directive support | no | yes | `--preproc-directives` | +| **Variable handling** | | | | +| Variable data class | `mkcap.Var` (flat attrs) | `metavar.Var` (validated prop dict) | | +| Scope-chain variable search | no | yes | group→suite→constituent→host | +| Variable promotion group→suite | no | yes | | +| Unit conversion | yes | yes | | +| Optional/active variables | yes (fully) | yes | both: local pointer + conditional ASSOCIATE | +| DDT library (first-class) | no | yes | `VarDDT` recursive chain | +| **Suite and cap generation** | | | | +| Suite definition (SDF XML) | yes | yes | Same XML format | +| Subcycle loops | yes | yes | | +| State machine in generated caps | no | yes | Runtime state enforcement | +| Static API module (dispatch switch) | yes | no | `ccpp_static_api.F90` | +| Host cap generation | no | yes | `_ccpp_cap.F90` | +| `ccpp_kinds.F90` | no | yes | | +| **Constituent/tracer support** | | | | +| Constituent variable management | no | yes | Auto-allocation, `ConstituentVarDict` | +| **Build system output** | | | | +| CMake/Makefile file-list snippets | yes | no | Six snippet files | +| Datatable XML (queryable) | no | yes | `ccpp_datafile.py` | +| Clean via datatable | no | yes | | +| **Documentation** | | | | +| HTML variable table | yes | no (stub, raises error) | `mkdoc.metadata_to_html` | +| LaTeX variable table | yes | no | `mkdoc.metadata_to_latex` | +| **Developer tools** | | | | +| Variable tracking diagnostic | yes | no | `ccpp_track_variables.py` | +| Fortran-to-metadata bootstrap | no | yes | `ccpp_fortran_to_metadata.py` | +| **Runtime API** | | | | +| In-memory database object | no | yes | `CCPPDatabaseObj` | +| **Debug / developer aids** | | | | +| Debug array-size checks in caps | yes (`--debug`) | no | | +| Namespace suffix for API name | yes (`--namespace`) | no | | +| **Configuration** | | | | +| Config mechanism | Python module (flexible) | CLI args only | | + +**Known gaps and corrections:** + +- Capgen's `--generate-docfiles` is declared in the CLI but raises + `CCPPError("not yet supported")` — documentation generation is unimplemented. +- Prebuild handles `TYPEDEFS_NEW_METADATA` for mixed old/new metadata deployments; + capgen has no equivalent because it only accepts the new format. +- Capgen validates Fortran source against metadata; prebuild trusts metadata and never + reads Fortran code. +- Capgen has no `--namespace` equivalent for the generated API module name. +- Capgen's `CCPPDatabaseObj` and datatable XML allow programmatic querying; prebuild + has no equivalent. +- Prebuild's static API pattern (single Fortran module with runtime dispatch) is absent + from capgen, which uses a different host-cap integration model. +- **Capgen cannot pass DDTs to group caps** — it passes everything as flat fields. + Despite considerable effort by multiple developers, this has not been fixed. This is + the primary reason capgen is being abandoned. +- **Capgen does not support multiple model instances in memory** (ensemble approach). + Prebuild's `initialized(200)` array handles this correctly. +- **Capgen does not own or allocate any data.** Wait — this is a prebuild characteristic. + Capgen *does* allocate data for physics-internal variables (variables used only within + the physics, not provided by the host model) at the suite level. Prebuild requires the + host model to provide and own all data, including any physics-internal scratch space. + +--- + +## 6. Build system integration + +### 6.1 How a host model invokes ccpp-prebuild + +Direct call (as in the test suite): +```bash +python ../../scripts/ccpp_prebuild.py \ + --config=ccpp_prebuild_config.py \ + --builddir=build \ + --suites=suite_A,suite_B \ + [--debug] [--namespace mymodel] +``` + +Typical CMake integration: +```cmake +# Run prebuild at configure time +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_prebuild.py + --config=${HOST_CCPP_PREBUILD_CONFIG} + --builddir=${CMAKE_CURRENT_BINARY_DIR} + --suites=${CCPP_SUITES} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + RESULT_VARIABLE PREBUILD_RESULT +) +if(NOT PREBUILD_RESULT EQUAL 0) + message(FATAL_ERROR "ccpp_prebuild.py failed") +endif() + +# Consume the generated snippet files +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_CAPS.cmake) # → ${CAPS} +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_SCHEMES.cmake) # → ${SCHEMES} +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_TYPEDEFS.cmake) # → ${TYPEDEFS} +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_API.cmake) # → ${API} + +add_library(ccpp_physics OBJECT ${CAPS} ${SCHEMES} ${API}) +``` + +### 6.2 How a host model invokes ccpp-capgen + +Direct call: +```bash +python scripts/ccpp_capgen.py \ + --host-files host_data.meta,host_model.meta \ + --scheme-files scheme1.meta,scheme2.meta \ + --suites suite_A.xml,suite_B.xml \ + --output-root ${BUILD_DIR}/ccpp \ + --host-name my_host \ + --kind-type kind_phys=REAL64 \ + --ccpp-datafile ${BUILD_DIR}/ccpp/datatable.xml +``` + +Typical CMake integration: +```cmake +# Run capgen at configure time +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_capgen.py + --host-files ${HOST_META_FILES} + --scheme-files ${SCHEME_META_FILES} + --suites ${SUITE_SDFS} + --output-root ${CMAKE_CURRENT_BINARY_DIR}/ccpp + --host-name ${HOST_MODEL_NAME} + --ccpp-datafile ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml + RESULT_VARIABLE CAPGEN_RESULT +) + +# Query the datatable for generated file lists +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_datafile.py + ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml + --suite-files + OUTPUT_VARIABLE SUITE_CAPS OUTPUT_STRIP_TRAILING_WHITESPACE +) +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_datafile.py + ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml + --host-files + OUTPUT_VARIABLE HOST_CAP OUTPUT_STRIP_TRAILING_WHITESPACE +) + +add_library(ccpp_physics OBJECT ${SUITE_CAPS} ${HOST_CAP}) +``` + +### 6.3 Available datatable query flags + +``` +--host-files → generated host cap .F90 files +--suite-files → generated suite cap .F90 files +--utility-files → generated utility .F90 files (e.g. ccpp_kinds.F90) +--ccpp-files → all generated .F90 files +--process-list → physics process types in the suite +--module-list → Fortran module names needed +--dependencies → scheme dependency files +--suite-list → configured suite names +--required-variables → variables required by all suites +--input-variables → input-only variables for a suite +--output-variables → output variables for a suite +--host-variables → variables provided by the host model +``` + +--- + +## 7. Key architectural differences + +### 7.1 Data model + +| Dimension | ccpp-prebuild | ccpp-capgen | +|---|---|---| +| Variable representation | `mkcap.Var` with plain Python attributes | `metavar.Var` with validated `__prop_dict` | +| Variable storage | Two flat `OrderedDict`s | Scope-chain `VarDictionary` tree | +| Container encoding | Encoded string: `MODULE_foo SCHEME_bar SUBROUTINE_baz` | Explicit class hierarchy | +| DDT handling | Encoded as string in `local_name`; helper regexes to extract | First-class `VarDDT` recursive chain | +| Variable matching | One batch `compare_metadata()` call | Incremental during suite analysis | +| Matching result | `bool` + side effects on `.target` / `.actions` | Rich `VarCompatObj` with transformation info | +| **Cap argument style** | **DDTs passed to group caps** | **Flat fields passed to group caps** | +| Subsetting location | At the scheme call site inside the group cap | Done at a higher level, before group cap | +| Data ownership | Host model owns all data including physics-internal | Capgen allocates physics-internal suite-level data | +| Multiple model instances | Yes — `initialized(200)` array, one flag per instance | No | +| Optional variable handling | Local pointer, conditionally associated | Same mechanism, but blocked by flat-field issue | + +### 7.2 Error handling + +| Aspect | ccpp-prebuild | ccpp-capgen | +|---|---|---| +| Style | `(success, result)` tuples + `logging.error()` | `CCPPError` / `ParseInternalError` exceptions | +| Collection | Errors accumulate via `logging`; `main()` checks success | Raised immediately at point of detection | +| Location info | Filename from context; line numbers sometimes | `ParseContext` objects with file + line number | +| User errors vs bugs | Not distinguished | `CCPPError` (user) vs `ParseInternalError` (programmer) | + +### 7.3 Extensibility + +| Aspect | ccpp-prebuild | ccpp-capgen | +|---|---|---| +| New metadata property | Add to `VALID_ITEMS` dict + `mkcap.Var` attribute | Add one `VariableProperty` entry + checker fn | +| New CCPP phase | Update `CCPP_STAGES` + regenerate static API template | Add one transition tuple to `CCPP_STATE_MACH` | +| New compatibility rule | Modify `var.compatible()` in `mkcap.py` | Extend `VarCompatObj` in `var_props.py` | +| New host model | Write a new Python config file | New `.meta` files + CLI invocation | + +### 7.4 Performance + +Prebuild generates caps for multiple suites in seconds. Capgen, on the same suite set +with the same physics, takes more than 10 minutes. Two independent causes: + +**Cause 1 — Repeated scope-chain traversal.** Every variable lookup in capgen traverses +a five-level `VarDictionary` parent chain (group → suite → constituent dict → host model +→ DDT dict) for every scheme argument in every group in every suite. Prebuild's +`compare_metadata()` does one flat dict lookup per standard name, once, and caches the +result in `var.container` and `var.target`. All subsequent use during Fortran generation +reads these cached attributes directly. + +**Cause 2 — Flat-field cap arguments.** This is likely the dominant cost. Capgen resolves +every scheme argument down to its individual flat field, generates a `use` statement and +an explicit argument for each one, and emits them in the generated Fortran. A DDT with +200 fields becomes 200 individual argument declarations, 200 `use` statements, and 200 +argument positions in the scheme call. Prebuild passes the DDT itself — one argument, +one `use` statement — and then subsets at the call site. + +**Consequence for correctness.** Passing flat fields in capgen also breaks optional +variable handling under Fortran compiler debugging flags. When a field inside a DDT is +conditionally allocated (optional), passing it as a flat field requires dereferencing the +DDT to extract the field — which the compiler will flag as an error if debugging is on +and the field happens to be unallocated. Prebuild avoids this entirely by passing the +DDT and using a local pointer at the scheme call site. + +### 7.5 Team comprehension and maintainability + +This is the critical real-world difference. `ccpp-prebuild` is understood by the whole +team because it is procedural Python: you can read `ccpp_prebuild.py` top-to-bottom and +follow what happens. The data structures are flat dicts; the control flow is linear. + +`ccpp-capgen` has a five-level class hierarchy, scope-chain dictionary lookups, +`VarCompatObj` carrying transformation state, `ConstituentVarDict` as a pluggable +scope-chain node, and a `StateMachine` with regex-based dispatch. No remaining team +member fully understands all of it. Development is extremely slow and risky. + +The failed effort to make capgen pass DDTs instead of flat fields is the concrete proof +point: three developers spent considerable time and could not fix it without fully +understanding the interplay between `VarDDT`, `DDTLibrary`, `VarDictionary` scope chains, +and the Fortran writer. This is the proximate reason for the redesign. + +--- + +## 8. Design considerations for the redesign + +The following observations from this analysis should inform the redesign: + +### 8.1 What to keep from prebuild +- Procedural, top-down control flow — easy to read and debug +- Config file as a Python module — extremely flexible without adding CLI arguments +- The static API pattern (`ccpp_static_api.F90` with runtime suite/group dispatch) — + proven, simple integration for the host model +- **DDT arguments in group caps** — pass DDTs, not flat fields; this is the core correctness + and performance requirement +- **Subsetting at the scheme call site** — group caps always receive full data; loop-bound + application and fixed-index extraction happen in the individual scheme call expressions + or via a local variable/pointer declared just before the call +- **Optional variable pattern** — local pointer declared in the group cap, conditionally + associated based on the `active` expression, then passed to the scheme; this is safe + under all compiler debugging modes +- The `initialized(N)` per-instance tracking — handles multiple simultaneous model + instances in memory (ensemble approach); `N` is the max number of instances +- **Framework-owned data needs a simpler design** — capgen's variable promotion and + `ConstituentVarDict` scope-chain approach is too complex; a cleaner mechanism for + framework-allocated physics-internal data is needed (to be designed) +- HTML and LaTeX documentation generation +- The six CMake/Makefile/shell snippet output files — simple and direct (can be revisited) + +### 8.2 What to keep from capgen +- Native `.meta` file parsing (eliminate the `metadata_parser.py` bridge entirely) +- Fortran source cross-validation (`check_fortran_against_metadata()`) — catches real bugs +- Rich compatibility reporting (`VarCompatObj`-style) — better error messages +- `ccpp_kinds.F90` generation — important for portability +- Datatable XML as output accounting (strictly better than six include files) +- `--preproc-directives` support +- Constituent variable support (needed for CAM-SIMA) +- State machine enforcement (optional feature, but architecturally clean) + +### 8.3 What to eliminate +- The `mkcap.Var` / `metavar.Var` duality — one variable class, natively reading `.meta` +- The `metadata_parser.py` bridge module — it exists only because of the old format +- The scope-chain `VarDictionary` hierarchy — replace with flat, explicit lookup: + one host dict, one scheme dict; no parent-chain traversal +- The five-level class inheritance (Suite → VarDictionary → ParseSource → ...) +- `ConstituentVarDict` as a scope-chain node — a simple explicit constituent registry suffices +- Capgen's variable promotion (group → suite level) — this complexity exists only because + capgen allocates physics-internal data; if the host always owns all data, promotion + is unnecessary +- Capgen's flat-field cap generation — DDT arguments must be the foundation + +### 8.4 Framework-owned data — open design question + +Capgen's variable promotion mechanism (promoting a variable from group scope to suite scope +when a later group needs it) and the `ConstituentVarDict` complexity exist because capgen +allocates and manages physics-internal data — variables used only within the physics, +not visible to the host model. This capability is **wanted** in the redesign: the host +model should not have to declare and own scratch variables that are purely internal to the +physics. + +The problem is not the concept but the implementation. Capgen's approach — weaving +framework-allocated variables into the `VarDictionary` scope chain and promoting them +upward — produces the complexity that made capgen unmaintainable. + +**Open question for the redesign:** What is a simpler mechanism for the framework to +allocate, own, and pass physics-internal variables? Candidate approaches (to be evaluated +with real-world examples): + +- A completely separate, flat "framework data" dictionary, distinct from the host variable + lookup, populated during analysis and passed explicitly to the caps as a dedicated + argument (e.g., a framework-managed DDT or allocatable array container). +- A simplified promotion concept: variables are statically promoted to the widest scope + that needs them during the analysis phase, but stored in a simple flat dict rather than + via a scope-chain lookup. +- Constituent variables (tracers) as a special sub-case with their own well-defined + allocation interface, separate from generic physics-internal data. + +This question will be revisited once real-world examples clarify how many and what kind of +physics-internal variables actually need to be managed. + +### 8.5 Critical design decisions for the redesign prompt + +1. **DDT cap arguments are non-negotiable.** Group caps must receive DDTs. The entire + subsetting, optional-variable, and performance story depends on this. + +2. **Data ownership**: host-owns-all (prebuild model) vs. generator-allocates-internals + (capgen model). This single decision determines whether variable promotion and + suite-level allocation are needed. + +3. **Integration pattern**: static API (prebuild style, `suite_name` + `group_name` dispatch) + vs. host cap (capgen style, separate host-side Fortran glue). Models currently using + each pattern depend on it. + +4. **Config mechanism**: Python module (prebuild style, flexible) vs. pure CLI + file lists + (capgen style, scriptable). The Python module config is very powerful for complex models. + +5. **DDT member access parsing**: `extract_parents_and_indices_from_local_name()` and + `extract_dimensions_from_local_name()` in `mkstatic.py` handle expressions like + `Atm(blk_no)%q(:,:,:,graupel_index)`. The redesign needs a clean, explicit design for + parsing and emitting these — not an afterthought regex patch. + +6. **Output accounting**: datatable XML (capgen) is the right answer. The six CMake snippet + files (prebuild) are redundant and harder to extend. + +7. **Multiple model instances**: the redesign must preserve the `initialized(N)` pattern + or an equivalent. The value of `N` may need to be configurable. + +8. **Backward compatibility of generated Fortran interfaces**: real-world model examples + will define exactly which naming conventions, argument orders, and module structures the + host models depend on. + +--- + +## 9. Real-world example: CCPP Single Column Model (SCM) + +*Source:* `EXT/ccpp-scm/` — uses `ccpp-prebuild`. + +The SCM is a horizontally degenerate model (always `im = 1`, no OpenMP threading) but +it compiles the largest set of suites in the CCPP ecosystem, making it the most complete +real-world picture of what prebuild must handle. + +**Scale:** 63 suites, 257 scheme files (137 scheme entries in config, many containing +multiple modules), 300 generated cap files, ~1,200+ host model variables, ~550 optional +(conditionally active) variables. + +--- + +### 9.1 The `TYPEDEFS_NEW_METADATA` bridge — the DDT accessor map + +This is the most important SCM-specific configuration. It maps each DDT type name to the +Fortran expression used to access an instance of that type from the host model's top-level +scope. It is what allows the code generator to convert a `local_name` like `tgrs` (declared +inside `GFS_statein_type`) into the cap argument expression +`physics%Statein%tgrs(...)`. + +```python +TYPEDEFS_NEW_METADATA = { + 'GFS_typedefs': { + 'GFS_diag_type' : 'physics%Diag', + 'GFS_control_type' : 'physics%Model', + 'GFS_cldprop_type' : 'physics%Cldprop', + 'GFS_tbd_type' : 'physics%Tbd', + 'GFS_sfcprop_type' : 'physics%Sfcprop', + 'GFS_coupling_type': 'physics%Coupling', + 'GFS_statein_type' : 'physics%Statein', + 'GFS_radtend_type' : 'physics%Radtend', + 'GFS_grid_type' : 'physics%Grid', + 'GFS_stateout_type': 'physics%Stateout', + 'GFS_typedefs' : '', + }, + 'CCPP_typedefs': { + 'GFS_interstitial_type': 'physics%Interstitial(cdata%thrd_no)', + 'CCPP_typedefs' : '', + }, + 'scm_type_defs': { + 'physics_type': 'physics', + 'scm_type_defs': '', + }, + 'ccpp_types': { + 'ccpp_t' : 'cdata', + 'ccpp_types': '', + 'MPI_Comm': '', + }, + # ... plus 8 more entries for physics-side modules (machine, radsw_param, etc.) +} +``` + +**How it works:** For a variable with `local_name = tgrs` declared in `GFS_statein_type`, +the generator looks up `'GFS_statein_type'` in the map, finds `'physics%Statein'`, and +constructs the target as `physics%Statein%tgrs`. For the thread-indexed interstitial DDT, +`physics%Interstitial(cdata%thrd_no)%` is produced automatically. + +This dictionary is the **entire** mechanism by which the prebuild bridge converts flat +metadata into correct DDT-member accessor expressions. It is a hand-maintained workaround +that the redesigned generator must **eliminate**: all information needed to derive these +accessor expressions is already present in the CCPP metadata, provided the metadata storage +model is designed correctly to capture the DDT hierarchy and instance/thread indexing. + +--- + +### 9.2 Host model DDT structure + +``` +! Module-level variables accessible globally: +physics (type physics_type, from module scm_type_defs) +cdata (type ccpp_t, from module ccpp_types) +one (integer parameter = 1, from module ccpp_types) + +! physics_type contains: +physics%Model → GFS_control_type (control parameters: integers, logicals, 1D arrays) +physics%Statein → GFS_statein_type (input atmospheric state: 2D/3D real arrays) +physics%Stateout → GFS_stateout_type (output tendencies) +physics%Sfcprop → GFS_sfcprop_type (surface properties: 2D real arrays) +physics%Coupling → GFS_coupling_type (coupling fields) +physics%Grid → GFS_grid_type (grid geometry) +physics%Tbd → GFS_tbd_type (to-be-determined / miscellaneous) +physics%Cldprop → GFS_cldprop_type (cloud microphysics properties) +physics%Radtend → GFS_radtend_type (radiation tendencies) +physics%Diag → GFS_diag_type (diagnostic output arrays) +physics%Interstitial(1:thrd_cnt) → GFS_interstitial_type (per-thread scratch space) +``` + +The interstitial DDT is an array indexed by thread number. Even though the SCM is +single-threaded, all caps use `physics%Interstitial(cdata%thrd_no)` (i.e., index 1). +This is the pattern that enables OpenMP parallelism in the full UFS models. + +--- + +### 9.3 The horizontal dimension in the SCM + +The SCM uses a **chunked** horizontal loop even though `im = 1`. The chunk mechanism is: + +```fortran +chunk_begin = physics%Model%chunk_begin(cdata%chunk_no) +chunk_end = physics%Model%chunk_end(cdata%chunk_no) +``` + +All 2D and 3D array slice expressions in caps use this pattern: +```fortran +physics%Statein%tgrs(chunk_begin:chunk_end, one:levs) +``` + +In the SCM, `chunk_begin = chunk_end = 1` always, but the pattern is general enough for +multi-column models. The `one` lower bound (a named integer constant = 1) is a framework +convention used consistently throughout all caps. + +--- + +### 9.4 Four categories of local variables in group caps + +Every group cap generates four categories of local variable declarations before its scheme +calls: + +**Category 1 — Loop bounds and scalars (always present):** +```fortran +integer :: chunk_begin, chunk_end +integer :: levs +chunk_begin = physics%Model%chunk_begin(cdata%chunk_no) +chunk_end = physics%Model%chunk_end(cdata%chunk_no) +levs = physics%Model%levs +``` + +**Category 2 — Fixed-index extractions (tracer indices, surface-level slices):** + +For a tracer `qgrs(:,:,ntqv)`: +```fortran +! No local variable declared — the expression is used inline at the call site: +call scheme_run(qv = physics%Statein%qgrs(chunk_begin:chunk_end, one:levs, physics%Model%ntqv), ...) +``` + +For a surface-level slice `prsi(:,1)`: +```fortran +call scheme_run(prsi_sfc = physics%Statein%prsi(chunk_begin:chunk_end, 1), ...) +``` + +The fixed index may be a literal integer (`1`) or a runtime scalar variable from a DDT +field (`physics%Model%ntqv`). Both are inlined at the call site. + +**Category 3 — Optional variable pointer arrays:** + +One pointer-array type and one pointer-array variable are declared for each optional +variable. They are dimensioned by thread count: +```fortran +type :: real_kind_phys_rank2_ptr_arr_type + real(kind_phys), dimension(:,:), pointer :: p => null() +end type real_kind_phys_rank2_ptr_arr_type +type(real_kind_phys_rank2_ptr_arr_type), dimension(1:cdata%thrd_cnt) :: sfc_wts_1_ptr_array +``` + +Before each scheme call that uses the variable, the condition is evaluated and the pointer +either associated or left null: +```fortran +if (physics%Model%lndp_type /= 0) then + sfc_wts_1_ptr_array(cdata%thrd_no)%p => & + physics%Coupling%sfc_wts(chunk_begin:chunk_end, one:physics%Model%n_var_lndp) +end if +``` + +Passed to the scheme as a keyword argument: +```fortran +call gfs_surface_generic_pre_run(..., sfc_wts=sfc_wts_1_ptr_array(cdata%thrd_no)%p, ...) +``` + +After the call, the pointer is nullified: +```fortran +if (physics%Model%lndp_type /= 0) then + nullify(sfc_wts_1_ptr_array(cdata%thrd_no)%p) +end if +``` + +**Category 4 — Unit conversion local variables:** + +Not present in the SCM (GFS uses consistent SI units throughout). When present in other +models, a local array is declared, populated before the call, and passed as the argument: +```fortran +real(kind_phys) :: converted_var(chunk_begin:chunk_end) +converted_var(:) = physics%Statein%source_field(chunk_begin:chunk_end) * conversion_factor +call scheme_run(..., target_arg=converted_var, ...) +``` + +--- + +### 9.5 Array size checks + +Every array argument — mandatory or optional — has a size check immediately before the +scheme call. The check uses `size()` and computes the expected size from dimension variables: + +```fortran +! Mandatory variable — outer condition is always .true. +if (.true.) then + if (size(physics%Statein%tgrs(chunk_begin:chunk_end, one:levs)) /= & + (chunk_end-chunk_begin+1)*(levs-one+1)) then + write(cdata%errmsg, '(a,i8,a,i8)') & + 'Detected size mismatch for variable tgrs: expected ', expected, ' but got ', actual + ierr = 1 + return + end if +end if + +! Optional variable — outer condition mirrors the active= expression +if (physics%Model%lndp_type /= 0) then + if (associated(sfc_wts_1_ptr_array(cdata%thrd_no)%p)) then + if (size(sfc_wts_1_ptr_array(cdata%thrd_no)%p) /= expected_size) then + ...error... + end if + end if +end if +``` + +--- + +### 9.6 The `initialized(200)` array and instance management + +```fortran +logical, dimension(200), save :: initialized = .false. +``` + +`cdata%ccpp_instance` is a 1-based integer assigned to each independent CCPP state object. +In an ensemble, each ensemble member gets a different instance number (1–200). The `init_cap` +sets `initialized(cdata%ccpp_instance) = .true.` at the end of successful init. The +`run_cap` checks `if (.not. initialized(cdata%ccpp_instance))` and aborts with an error +if init was never called for that instance. The `final_cap` resets the flag to `.false.`. + +The value 200 is hardcoded — it is the maximum supported number of simultaneous model +instances. This could be made configurable. + +--- + +### 9.7 Suite and group cap hierarchy + +Three-level cap hierarchy: + +``` +ccpp_static_api.F90 (module ccpp_static_api) + → dispatches by suite_name + optional group_name + → owns physics, cdata, constants via module use + → calls suite-level caps: + +ccpp_scm_gfs_v16_cap.F90 (module ccpp_scm_gfs_v16_cap) + → aggregates arguments from all groups + → calls group caps in order per phase: + +ccpp_scm_gfs_v16_time_vary_cap.F90 (module ccpp_scm_gfs_v16_time_vary_cap) +ccpp_scm_gfs_v16_radiation_cap.F90 (module ccpp_scm_gfs_v16_radiation_cap) +ccpp_scm_gfs_v16_phys_ps_cap.F90 (module ccpp_scm_gfs_v16_phys_ps_cap) +ccpp_scm_gfs_v16_phys_ts_cap.F90 (module ccpp_scm_gfs_v16_phys_ts_cap) +``` + +Each level is a pure Fortran module. Argument passing is explicit keyword-argument style +at every level; no implicit global data (except in the static API, which uses `use`). + +--- + +### 9.8 Static API: module-level variable ownership + +The static API module uses all host-model modules and accesses their variables at module +scope. It does **not** take host data as subroutine arguments — instead it fills the +group cap arguments from its own module-use-associated variables: + +```fortran +module ccpp_static_api + use scm_type_defs, only: physics + use ccpp_types, only: cdata, one + use scm_physical_constants, only: con_g, con_pi, con_t0c, ... + use gfs_typedefs, only: ltp + use ccpp_scm_gfs_v16_cap, only: scm_gfs_v16_run_cap, ... + ... +contains + subroutine ccpp_physics_run(cdata, suite_name, group_name, ierr) + ! cdata passed in, others accessed from module scope + select case (to_lower(trim(suite_name))) + case ('scm_gfs_v16') + if (present(group_name)) then + select case (to_lower(trim(group_name))) + case ('phys_ps') + ierr = scm_gfs_v16_phys_ps_run_cap(one=one, physics=physics, cdata=cdata, ...) + ... + end select + else + ierr = scm_gfs_v16_run_cap(one=one, physics=physics, cdata=cdata, ...) + end if + case ('scm_gfs_v17_p8') + ... + end select + end subroutine +end module +``` + +This design means the static API file must be recompiled whenever any host-model module +changes (because it `use`s them), and it must be regenerated whenever suites change. +Its location in the **source tree** (not build tree) is a deliberate SCM design choice: +the file is committed to the repository as a generated artifact. + +--- + +### 9.9 Build system + +Prebuild runs at **cmake configure time** via `execute_process()`, before any compilation +starts. This is unusual but simplifies the cmake dependency graph. + +```cmake +execute_process( + COMMAND ccpp/framework/scripts/ccpp_prebuild.py + --config=ccpp/config/ccpp_prebuild_config.py + --suites=${CCPP_SUITES} + --builddir=${CMAKE_CURRENT_BINARY_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../.. + OUTPUT_FILE ${PROJECT_BINARY_DIR}/ccpp_prebuild.out + ERROR_FILE ${PROJECT_BINARY_DIR}/ccpp_prebuild.err +) +include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_CAPS.cmake) # → ${CAPS} +include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_SCHEMES.cmake) # → ${SCHEMES} +include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_TYPEDEFS.cmake) # → ${TYPEDEFS} +include(scm/src/CCPP_STATIC_API.cmake) # → ${API} +``` + +**Suite selection:** If `CCPP_SUITES` is not set by the user, a helper script +`suite_info.py` selects a compiler-appropriate subset. The full set of 63 suites is +used for production; subsets speed up development builds. + +--- + +### 9.10 Observations relevant to the redesign + +1. **`TYPEDEFS_NEW_METADATA` is a workaround that the redesign must eliminate.** The + DDT accessor information (which type lives at which accessor path) can be fully derived + from the CCPP metadata itself, given a well-designed metadata storage model. The + redesign must derive DDT accessor expressions automatically from the metadata rather + than requiring a separate hand-maintained dictionary. This is one of the primary + motivations for the new metadata storage design. + +2. **Three-level cap hierarchy (group → suite → static API) should be preserved.** + It provides clean separation: group caps are independently testable, suite caps + aggregate phases, the static API is the single host-callable entry point. + +3. **The static API's module-level `use` of host data is model-specific.** In models + where host data is not module-level (e.g., passed as subroutine arguments), the + static API pattern changes. The SCM is the simplest case because `physics` and `cdata` + are global module variables. + +4. **Instance and thread indexing are two orthogonal dimensions of host data access.** + Host model data uses two distinct indexing patterns that must be handled correctly: + + - **Regular state data** (Statein, Stateout, Sfcprop, etc.): dimensioned by instance + number — `physics%Statein(ccpp_instance_number)%array(1:horizontal_dimension, 1:vertical_dimension, ...)`. + In models supporting multiple in-memory model instances (ensemble), the top-level + DDT is an array indexed by `cdata%ccpp_instance`. + + - **Interstitial (per-thread scratch) data**: dimensioned by both instance and thread — + `physics%Interstitial(ccpp_instance_number, ccpp_thread_number)%array(1:horizontal_loop_extent, ...)`. + Critically, the horizontal dimension of interstitial arrays is sized to + `horizontal_loop_extent` (one OpenMP thread's chunk), not `horizontal_dimension` + (the full column count). `max_number_of_threads` instances are allocated per model + instance. Interstitial data can only be used during the **run phase** — this is a + known limitation of ccpp-prebuild that the redesign should address or at minimum + preserve explicitly. + +5. **Optional variable pointer arrays dimensioned by thread count** are the current + solution to thread-safe optional variable handling. This pattern is verbose (one + derived type + one array per optional variable per cap function) but correct. + The redesign could simplify this. + +6. **~550 optional variables in this model.** Optional/conditional variables are not + a corner case — they are a first-class feature. The redesign must handle them + efficiently and correctly. + +7. **Array size checks are debug-only and should not appear in the redesign by default.** + In prebuild they are only generated when the `--debug` flag is passed. The redesigned + generator should not produce them in normal mode — out-of-bounds access is caught at + runtime by compiler flags (e.g., `-fcheck=bounds` with gfortran, `-check bounds` with + ifort). The 12,991-line group cap is partly a consequence of generating these checks + unconditionally in the debug mode artifact examined here. + +8. **No unit conversions appear in GFS/SCM.** Unit conversion infrastructure must be + present in the redesign but the GFS physics package is self-consistent in units. + Unit conversions are more relevant for other host models. + +9. **The `one` constant** (integer parameter = 1) is passed as an explicit argument + everywhere and used as the lower bound in all array slices. This is a framework + convention. The redesign should decide whether this convention is preserved or + whether array lower bounds are handled differently. + +10. **Subcycles produce actual Fortran `do` loops inside the generated group cap.** + The loop from `1` to `cdata%loop_max` is generated directly in the cap function, + not left to the host model: + ```fortran + cdata%loop_max = 2 + do cdata%loop_cnt = 1, cdata%loop_max + call scheme_A_run(...) + if (ierr /= 0) return + call scheme_B_run(...) + if (ierr /= 0) return + end do + ``` + `cdata%loop_max` is set at the start of the subcycle block (from the `loop=` attribute + in the SDF XML) and `cdata%loop_cnt` is the current iteration counter, both visible + to schemes via the `ccpp_t` DDT. + +--- + +## 10. Real-world example: CAM-SIMA (capgen) + +*Source:* `EXT/cam-sima/` — uses `ccpp-capgen`. + +CAM-SIMA is the only model currently using capgen. It is still primarily a research model. +Unlike the SCM it uses a full 3D grid with OpenMP parallelism, but exposes host model +data as flat module variables rather than DDTs in the metadata layer. This example reveals +both what capgen can do and where it fundamentally fails. + +**Scale:** 1 suite (`cam7`), 2 run groups (`physics_before_coupler`, `physics_after_coupler`), +~75 scheme calls, 18 host `.meta` files, 893-line host cap, 2865-line suite cap. + +--- + +### 10.1 Suite structure + +`suite_cam7.xml` has two groups and no subcycles: + +| Group | Schemes (approx.) | Purpose | +|---|---|---| +| `physics_before_coupler` | 52 scheme calls | Cloud fraction, energy checks, dry adiabatic adjustment, Zhang-McFarlane deep convection full cycle, constituent tendency application | +| `physics_after_coupler` | ~20 scheme calls | Tropopause diagnostics, gravity wave drag (7 parameterizations + diagnostics), tendency application, energy consistency | + +CCPP phases in use: register, initialize, timestep_initial, run (per group), timestep_final, finalize. + +--- + +### 10.2 Host model variable structure + +**18 host `.meta` files, all of type `module`.** There are no `host` or `ddt` table types +anywhere. All host variables are flat scalars or arrays in Fortran modules. + +CAM-SIMA does **not** expose its physics DDTs (`phys_state`, `phys_tend`, `cam_in`, etc.) +through metadata. These types exist in `physics_types.F90` but have no `.meta` file. +The generated host cap accesses them directly via `use physics_types, only: phys_state, ...` +and passes individual DDT members as flat keyword arguments: +```fortran +! In cam_ccpp_cap.F90 — direct access to non-metadataized DDT members: +call cam7_physics_before_coupler(..., pint=phys_state%pint, t=phys_state%t, & + dtdt_total=phys_tend%dtdt_total, landfrac=cam_in%landfrac, ...) +``` + +This means capgen has no knowledge of how host data is structured. The host cap is +partly machine-generated and partly depends on manually wiring non-metadataized sources. +**This is a fundamental architectural gap** — changes to `physics_types` are invisible +to the framework. + +**Key host variables by module:** + +| Module | Key variables | +|---|---| +| `physics_grid` | `columns_on_task` (horizontal_dimension), `col_start`, `col_end`, lat, lon, area | +| `vert_coord` | `pver` (vertical_layer_dimension), `pverp` (vertical_interface_dimension) | +| `physconst` | ~35 physical constants, all `protected = True` | +| `cam_constituents` | `num_advected` (count of advected tracers) | +| `spmd_utils` | `mpicom`, `masterproc`, `npes`, `iam` | + +No instance indexing (`physics(1)`) and no thread-indexed DDTs appear — CAM-SIMA uses +a fundamentally different data model from the GFS/SCM stack. + +--- + +### 10.3 The two-cap architecture + +Capgen generates two distinct Fortran files: + +**`cam_ccpp_cap.F90` — the host cap (893 lines)** +- Module `cam_ccpp_cap` +- Imports non-metadataized host variables directly via `use physics_types`, `use physconst`, etc. +- Manages the constituent object (`ccpp_model_constituents_t`) — registration, initialization, gather/scatter, index lookup +- Public subroutines: `cam_ccpp_physics_run`, `cam_ccpp_physics_initialize`, etc. — the entry points the host model calls +- Dispatches to the suite cap, passing ~61–76 flat keyword arguments + +**`ccpp_cam7_cap.F90` — the suite cap (2865 lines)** +- Module `ccpp_cam7_cap` +- No host-specific imports — knows nothing about `physics_types`, `phys_state`, etc. +- All arguments are flat scalars and arrays, fully matched to metadata standard_names +- Contains all scheme calls, suite-level persistent variables, local temporaries, state machine +- The suite cap could in principle be used with any host model that provides the same standard names + +This two-cap split is **architecturally correct**: it separates host-specific binding +from physics-neutral dispatch. The redesign should preserve this separation. + +--- + +### 10.4 The flat-field argument problem — concrete evidence + +The run-phase subroutines expose the core problem with capgen's approach directly: + +```fortran +subroutine cam7_physics_before_coupler(errflg, errmsg, col_start, col_end, pver, dtime, & + gravit, pint, te_ini_dyn, teout, amiroot, iulog, ptend_s, temp, dtdt_total, cpair, & + lagrang, layer_surf, layer_toa, interface_surf, interface_toa, ncnst, piln, pmid, pdel, & + rpdel, qv, carr, cprops, rair, zvir, zi, zm, cp_or_cv_dycore, u, v, pintdry, phis, & + te_cur_phys, te_cur_dyn, tw_cur, latice, latvap, energy_formula_physics, & + energy_formula_dycore, cappa, q_tend, const_tend, qmin, pverp, cpwv, cpliq, rh2o, lat, & + long, pblh, mcon, tpert, dlf, rprd, ql, rliq, landfrac, cpair3, ttend_dp, tmelt, & + top_lev, ke, ke_lnd, cldfrc, domomtran, momcu, momcd, il1g, nstep, & + dudt_total, dvdt_total, fracis, dpdry, ps) +``` + +**61 dummy arguments for one group cap.** `physics_after_coupler` has 76. These are +individual flat arrays and scalars — no DDT in sight. This is exactly the problem that +three developers failed to fix: in the GFS/UFS context, this would be 1,200+ arguments. +The GFS physics stack simply cannot be connected to capgen in its current form. + +In contrast, the prebuild equivalent for the same data would pass `physics` (one DDT argument) +and `cdata` — two arguments covering hundreds of variables. + +--- + +### 10.5 Suite-level persistent variables — the framework-owned data pattern + +The suite cap allocates and owns arrays that persist across group calls within a timestep. +These are allocated in `cam7_initialize` and deallocated in `cam7_finalize`: + +```fortran +! Suite-level persistent (allocated in initialize, freed in finalize): +real(kind_phys), allocatable :: windu_tend(:,:) ! GW drag u-tendency accumulator +real(kind_phys), allocatable :: windv_tend(:,:) ! GW drag v-tendency accumulator +real(kind_phys), allocatable :: scaling_dycore(:,:) ! energy scaling factor +real(kind_phys), allocatable :: tend_te_tnd(:) ! energy tendency accumulator +real(kind_phys), allocatable :: tend_tw_tnd(:) ! water tendency accumulator +real(kind_phys), allocatable :: temp_ini(:,:) ! temperature saved at timestep start +real(kind_phys), allocatable :: z_ini(:,:) ! height saved at timestep start +real(kind_phys), allocatable :: flx_vap(:), flx_cnd(:), flx_ice(:), flx_sen(:) +logical, allocatable :: doconvtran(:) ! per-constituent convection flag +type(coords1d) :: p ! pressure coordinate DDT for GW drag +``` + +These are physics-internal variables — the host model does not know about them, does not +own them, and does not need to. This is the capgen "data ownership" model: the suite cap +is the data owner for variables that only matter within the physics. + +**This pattern is correct and desirable.** The complexity in capgen comes not from the +concept but from how these variables are discovered during analysis (scope-chain promotion) +and passed around (via VarDictionary). The redesign needs a simpler mechanism to achieve +the same result: statically enumerate physics-internal variables during analysis and have +the suite cap own them as named allocatables. + +During the run phase, suite-level persistent arrays are subsetted when passed to schemes: +```fortran +call gw_common_run(..., windu_tend=windu_tend(col_start:col_end, 1:pver), ...) +``` + +Run-phase local temporaries (e.g., `cape`, `cme`, `mu`, `md`) are allocated at function +entry and deallocated at exit: +```fortran +allocate(cape(col_start:col_end)) +... +call zm_convr_run(..., cape=cape, ...) +... +deallocate(cape) +``` + +These temporaries use `col_start` as the lower bound so that assumed-shape dummy arguments +in schemes see a 1-based array — a subtle but important detail. + +--- + +### 10.6 Horizontal chunking model + +CAM-SIMA uses `col_start`/`col_end` (passed as arguments to every run subroutine) to +define the current horizontal chunk: + +```fortran +ncol = col_end - col_start + 1 +``` + +Schemes declare `horizontal_loop_extent` and receive `ncol`. The horizontal dimension +in the host (storage dimension) is `columns_on_task`. The subsetting from storage to +loop extent happens at the boundary between host cap and suite cap — the host cap +passes the right subsections: + +```fortran +! In cam_ccpp_cap.F90: +call cam7_physics_before_coupler(..., col_start=col_start, col_end=col_end, & + pmid=phys_state%pmid, ...) ! full arrays passed; suite cap subsets internally +``` + +Inside the suite cap, persistent arrays are subsetted explicitly when passed to schemes: +```fortran +windu_tend(col_start:col_end, 1:pver) +``` +Local temporaries allocated as `allocate(cape(col_start:col_end))` are already +correctly sized and passed as assumed-shape `(:)`. + +--- + +### 10.7 State machine + +The suite cap has a character module variable tracking lifecycle state: + +```fortran +character(len=16) :: ccpp_suite_state = 'uninitialized' +``` + +Transitions: `uninitialized` → register → `uninitialized` → initialize → `initialized` +→ timestep_initial → `in_time_step` → (run, no state change) → timestep_final → +`initialized` → finalize → `uninitialized`. + +Each phase entry point checks the expected prior state: +```fortran +if (trim(ccpp_suite_state) /= 'in_time_step') then + errflg = 1 + write(errmsg, '(3a)') "Invalid initial CCPP state, '", trim(ccpp_suite_state), & + "' in cam7_physics_before_coupler" + return +end if +``` + +Non-run phases also include an OpenMP thread guard: +```fortran +#ifdef _OPENMP + if (omp_get_thread_num() > 1) then + errflg = 1 + errmsg = "Cannot call initialize routine from a threaded region" + return + end if +#endif +``` + +The state machine is simple, complete, and useful. The redesign should preserve it. + +--- + +### 10.8 Constituent variable handling + +CAM-SIMA demonstrates the full constituent lifecycle: + +```fortran +! In cam_ccpp_cap.F90: +type(ccpp_model_constituents_t), target :: cam_constituents_obj + +! Registration (scheme-declared constituents): +call suite_cam7_constituents_num_consts(num_consts) +call suite_cam7_constituents_const_name(iconst, const_name) +call cam_constituents_obj%new_field(const_name, ...) + +! Initialization (host-declared constituents like water vapor): +cam_model_const_stdnames(1) = "water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water" +call cam_constituents_obj%new_field(cam_model_const_stdnames(1), ...) + +! Per-timestep gather from host: +call cam_ccpp_gather_constituents(phys_state%q, ...) + +! Passing to suite cap: +call cam7_physics_before_coupler(..., + qv = cam_constituents_obj%vars_layer(:, :, cam_model_const_indices(1)), + carr = cam_constituents_obj%vars_layer, + cprops = cam_constituents_obj%const_metadata, ...) + +! Per-timestep scatter back to host: +call cam_ccpp_update_constituents(phys_state%q, ...) +``` + +The suite cap sees constituents as: +- `carr(:,:,:)` — the full rank-3 constituent array (ncol, nlev, ncnst) +- `qv(:,:)` — water vapor slice extracted in the host cap: `cam_constituents_obj%vars_layer(:,:,cam_model_const_indices(1))` +- `cprops(:)` — array of `ccpp_constituent_prop_ptr_t` metadata objects +- `doconvtran(1:ncnst)` — suite-level logical array set by scheme init indicating which constituents are convected + +This constituent API is sophisticated and worth preserving or improving in the redesign. + +--- + +### 10.9 Known defects in the capgen output + +**Repeated scheme init/final calls.** Capgen generates one init call per occurrence of +a scheme name in the XML, without deduplication: +- `qneg_init` called 5 times (once per `qneg` entry in the suite XML) +- `qneg_timestep_final` called 5 times +- `check_energy_chng_init` called twice +- `save_ttend_from_convect_deep_timestep_init` called 3 times + +If these routines have internal state, allocations, or side effects, this is a correctness +defect. The redesign must deduplicate init/final calls by unique scheme name. + +**Unit conversion embedded silently in the cap.** Before `zm_conv_convtran_run`: +```fortran +dpdry_local(:,1:pver) = 1.0E-2_kind_phys * dpdry(:,1:pver) ! Pa → hPa +``` +This is generated from the metadata units mismatch but appears as an opaque transform +in the cap. The redesign should make this visible (e.g., a comment naming the standard +name, the source units, and the target units). + +--- + +### 10.10 Build system — capgen invocation + +Capgen is invoked from Python (`cam_autogen.py`), not from cmake: + +```python +from ccpp_capgen import capgen +capgen_db = capgen(run_env, return_db=True) +``` + +This is a programmatic API call, not a subprocess. The `CCPPDatabaseObj` returned +(`capgen_db`) is then used directly in Python to query scheme lists, constituent names, +and file paths — avoiding the datatable XML query step that cmake-based invocations need. + +Output files consumed by the build: +- `cam_ccpp_cap.F90` — compiled into the atmosphere component +- `ccpp_cam7_cap.F90` — compiled into the atmosphere component +- `ccpp_kinds.F90` — compiled into the atmosphere component +- Utility files from `ccpp_framework/src/` (copied to build dir) +- `ccpp_datatable.xml` — queried by the build system for file lists + +--- + +### 10.11 Observations relevant to the redesign + +1. **The two-cap split (host cap + suite cap) is the right architecture.** It cleanly + separates host-specific binding from physics-neutral dispatch. The redesign must + preserve this. + +2. **Flat-field arguments in the suite cap are the critical failure.** 61–76 dummy + arguments per run subroutine is already large for a research model; for UFS/GFS + with 1,200+ variables it is completely infeasible. The redesign must pass DDTs. + +3. **The CAM-SIMA host does not use DDTs in metadata.** All host variables are flat + module variables. This is a fundamentally different host model architecture from + GFS/SCM. The redesign must support both styles: flat-module hosts (CAM-SIMA) and + deep-DDT hosts (GFS/SCM). + +4. **Non-metadataized variables hardwired into the host cap is a serious gap.** + `phys_state`, `phys_tend`, `cam_in` from `physics_types` have no `.meta` files. + The host cap accesses them directly. This means the framework cannot verify or + track these variables. The redesign should either require full metadata coverage + or have an explicit mechanism for declaring non-metadataized pass-through variables. + +5. **Suite-level persistent variables (framework-owned data) work well in practice.** + `windu_tend`, `scaling_dycore`, `temp_ini`, etc. are owned by the suite cap, invisible + to the host, and persist across group calls. This is the right pattern for + physics-internal state. The redesign needs this but with a simpler discovery mechanism + than capgen's scope-chain promotion. + +6. **Deduplicate init/final calls.** The redesign must deduplicate `_init`, `_finalize`, + `_timestep_init`, and `_timestep_final` calls by unique scheme name (not by occurrence + in the XML). + +7. **The constituent API in the host cap is comprehensive.** The `ccpp_model_constituents_t` + object with its register/init/gather/scatter/index API is sophisticated and should be + preserved or improved. + +8. **The suite-variables introspection subroutine** (`ccpp_physics_suite_variables`, + enumerating 83 standard names as inputs/outputs) is a useful capability for build + system integration and should be in the redesign. + +9. **The programmatic Python API** (`capgen(run_env, return_db=True)`) is valuable + for hosts like CAM-SIMA that invoke the generator from Python. The redesign should + support both CLI and programmatic invocation. + +10. **Unit conversions must be annotated in the generated cap**, not silently embedded + as magic-number multiplications. A comment with source units, target units, and the + standard name involved is the minimum. + +11. **The horizontal chunking model** (`col_start`/`col_end` as explicit arguments, + `ncol = col_end - col_start + 1` computed at entry) works and is clean. Suite-level + persistent arrays are allocated full-size and subsetted at call sites. + +12. **No optional variables in this model.** CAM-SIMA does not exercise optional/active + variable handling. This feature must be in the redesign but is not demonstrated here. + +--- + +## 11. Real-world example: UFS Weather Model (prebuild) + +The UFS Weather Model is the most complex and production-critical of the three examples. +It is a fully-coupled, 3-D operational NWP model. The CCPP physics is used in the +atmospheric component (`UFSATM`). Unlike the SCM (column model, process-split only) and +CAM-SIMA (capgen, flat-field arguments), UFS uses prebuild in a 3-D blocked/threaded +configuration that is architecturally distinct from both prior examples. + +The two suites analyzed here are: +- `FV3_GFS_v17_coupled_p8` — the primary operational GFS suite +- `FV3_GFS_v17_coupled_p8_ugwpv1` — a variant replacing `unified_ugwp` with `ugwpv1` + +The ugwpv1 suite is structurally identical to the base suite except for the `phys_ps` +group (4 extra scheme calls), so all observations below apply to both. + +--- + +### 11.1 Suite structure + +The primary suite has 5 groups: + +| Group | Subcycles | Scheme calls | Phase called | +|-------|-----------|-------------|--------------| +| `time_vary` | 1 | 4 | timestep_init (domain-level, no blocking) | +| `radiation` | 1 | 8 | run (block/thread loop) | +| `phys_ps` | 3 (loop=1, loop=2, loop=1) | 21 | run (block/thread loop) | +| `phys_ts` | 3 (loop=1, loop=1, loop=1) | 12 | run (block/thread loop) | +| `stochastics` | 1 | 2 | run (block/thread loop) | + +The `time_vary` group is the only one called at timestep_init/finalize. All other groups +are called from the run phase via the OpenMP blocked loop. This is a fundamentally +different usage pattern from SCM (which runs everything sequentially) and CAM-SIMA +(which has no run phase at all for the groups analyzed). + +The `phys_ps` group has a surface iteration subcycle with `loop="2"`, which generates an +actual Fortran `do` loop in the cap body: +```fortran +! Start of next subcycle +cdata%loop_max = 2 +do cdata%loop_cnt = 1, cdata%loop_max + ! ... sfc_diff, sfc_nst, noahmpdrv, sfc_land, sfc_cice, sfc_sice ... +end do +``` + +--- + +### 11.2 Cap hierarchy and scale + +The three-level hierarchy is preserved from prebuild: + +``` +ccpp_static_api.F90 (627 lines) ← suite+group name dispatch + ↓ +ccpp_fv3_gfs_v17_coupled_p8_cap.F90 (363 lines) ← calls all group caps in order + ↓ +ccpp_fv3_gfs_v17_coupled_p8_time_vary_cap.F90 (1404 lines) +ccpp_fv3_gfs_v17_coupled_p8_radiation_cap.F90 (967 lines) +ccpp_fv3_gfs_v17_coupled_p8_phys_ps_cap.F90 (4226 lines) ← 200 optional ptr arrays +ccpp_fv3_gfs_v17_coupled_p8_phys_ts_cap.F90 (1953 lines) +ccpp_fv3_gfs_v17_coupled_p8_stochastics_cap.F90 (443 lines) +``` + +The ugwpv1 variant generates another 10,220 lines of largely redundant code (identical +caps with one suite-name prefix change and minor scheme-list differences). Total for both +suites: 18,333 lines of generated Fortran. + +This redundancy is a key motivation for the redesign: suite variants that share groups +should not regenerate identical cap code. The redesign should support group-level cap +sharing across suite variants. + +--- + +### 11.3 Host model DDT structure + +All host data lives in `CCPP_data.F90` as module-level `save, target` variables: + +```fortran +type(GFS_control_type) :: GFS_control ! config/control +type(GFS_statein_type) :: GFS_statein ! atmospheric state in +type(GFS_stateout_type) :: GFS_stateout ! atmospheric state out +type(GFS_grid_type) :: GFS_grid ! grid geometry +type(GFS_tbd_type) :: GFS_tbd ! temporal interp data +type(GFS_cldprop_type) :: GFS_cldprop ! cloud properties +type(GFS_sfcprop_type) :: GFS_sfcprop ! surface properties +type(GFS_radtend_type) :: GFS_radtend ! radiation tendencies +type(GFS_coupling_type) :: GFS_coupling ! coupling fields +type(GFS_diag_type) :: GFS_intdiag ! diagnostics +type(GFS_interstitial_type), allocatable (:) :: GFS_interstitial ! scratch, per thread +``` + +Plus three `ccpp_t` instances for different levels of parallelism (see §11.5). + +This is structurally similar to the SCM's `physics` DDT hierarchy, but with one key +difference: all DDTs are at the same flat level rather than nested (no `physics%Statein`, +only `GFS_statein`). Each DDT maps to a distinct functional role. + +The `GFS_typedefs.F90` file (not auto-generated) defines all DDT types along with ~30 +physical constants (`con_pi`, `con_g`, `con_rd`, etc.) that also appear in the metadata. + +--- + +### 11.4 DDT arguments in the cap chain + +The static API imports all DDTs and physical constants from `CCPP_data` and `GFS_typedefs` +via `use` statements, then passes them as named arguments to group cap functions. This is +the full DDT-argument pattern that prebuild implements: + +```fortran +! In ccpp_static_api.F90: +use ccpp_data, only: gfs_control, gfs_statein, gfs_sfcprop, ... +use gfs_typedefs, only: con_pi, con_g, con_rd, ... + +ierr = fv3_gfs_v17_coupled_p8_phys_ps_run_cap( & + one=one, gfs_control=gfs_control, cdata=cdata, & + gfs_statein=gfs_statein, gfs_sfcprop=gfs_sfcprop, & + con_g=con_g, con_pi=con_pi, ... & + gfs_interstitial=gfs_interstitial) +``` + +The group cap receives these as typed `intent(*), target` dummy arguments and uses them +directly to construct call-site subsections. This means **the group cap is fully portable +— it does not use any host module directly**, only what it receives as arguments. + +The `target` attribute is required because the cap creates pointer sections of these DDTs +(array subsections via pointer assignment) when handling optional variables. + +--- + +### 11.5 The dual cdata architecture + +UFS uses two distinct sets of `ccpp_t` handles with different scopes: + +**Domain-level (`cdata_domain`)**: Used for non-run phases (init, finalize, time_vary +timestep_init/finalize). Called once per step, no blocking: +```fortran +cdata_domain%blk_no = 1; cdata_domain%chunk_no = 1 +cdata_domain%thrd_no = 1; cdata_domain%thrd_cnt = 1 +``` + +**Block/thread-level (`cdata_block(nb, nt)`)**: Used for run phase (radiation, phys_ps, +phys_ts, stochastics). Allocated as a 2-D array `(1:nblks, 1:nthrdsX)` where `nthrdsX` +accounts for non-uniform last-block sizing: +```fortran +cdata_block(nb,nt)%blk_no = nb +cdata_block(nb,nt)%chunk_no = nb ! block number = chunk number +cdata_block(nb,nt)%thrd_no = nt +cdata_block(nb,nt)%thrd_cnt = nthrdsX +``` + +The redesign must support this dual cdata usage: a single `cdata` handle for domain-level +phases and a 2-D array of handles for blocked run phases. + +--- + +### 11.6 OpenMP threading model + +Non-run phases allow internal threading in physics schemes: +```fortran +GFS_control%nthreads = nthrds ! all N threads available to physics +call ccpp_physics_timestep_init(cdata_domain, ...) +``` + +Run phase uses all threads for blocking, so physics must not spawn additional threads: +```fortran +GFS_control%nthreads = 1 ! no internal threading allowed +!$OMP parallel num_threads(nthrds) ... +!$OMP do schedule(dynamic,1) +do nb = 1, nblks + call GFS_Interstitial(nt)%create(ixs=chunk_begin(nb), ixe=chunk_end(nb), model=GFS_control) + call ccpp_physics_run(cdata_block(nb,nt), group_name="phys_ps", ...) + call GFS_Interstitial(nt)%destroy(GFS_control) +end do +!$OMP end do +!$OMP end parallel +``` + +The `nt = omp_get_thread_num()+1` pattern (1-based thread index) is used throughout. +Each thread owns one `GFS_Interstitial(nt)` and one `cdata_block(nb,nt)` per block +iteration. The dynamic schedule means different threads process different blocks at +different times, which is why the interstitial must be created/destroyed per-iteration +rather than pre-allocated per-thread. + +--- + +### 11.7 Horizontal dimension: the chunk_begin/chunk_end pattern + +For non-run phases, the full horizontal dimension is used at every call site: +```fortran +tgrs(one:gfs_control%ncols, one:gfs_control%levs) +``` + +For run phases, the chunk range is looked up from the control DDT using the block number: +```fortran +tgrs(gfs_control%chunk_begin(cdata%chunk_no) : gfs_control%chunk_end(cdata%chunk_no), & + one:gfs_control%levs) +``` + +The chunk size (horizontal extent `im`) is retrieved as: +```fortran +im = gfs_control%blksz(cdata%blk_no) +``` + +`blksz(nb)` handles **non-uniform block sizes**: the last block may be smaller than the +others if the domain size is not divisible by the number of blocks. The `chunk_begin`/ +`chunk_end` arrays (indexed by chunk number = block number) give the global offset range. + +This is a cleaner pattern than SCM's `chunk_begin`/`chunk_end` as explicit dummy +arguments, because UFS looks them up from the already-passed `gfs_control` DDT. + +**Critical implication for the redesign**: The subsetting pattern `(chunk_begin:chunk_end)` +appears at every single array call site in the run phase — literally hundreds of times in +the phys_ps cap alone. This boilerplate is generated by prebuild from the metadata. In +the redesign, this subsetting must remain at the call site (not higher up) to allow each +thread to process its own chunk independently. + +--- + +### 11.8 The GFS_interstitial — pointer-based scratch DDT + +`GFS_interstitial_type` (defined in `CCPP_typedefs.F90`) is a DDT where **every field is +a pointer**, initialized to null: +```fortran +type GFS_interstitial_type + real(kind_phys), pointer :: adjsfculw_land(:) => null() + real(kind_phys), pointer :: del(:,:) => null() + ! ... ~200+ pointer fields +end type +``` + +This is dramatically different from the SCM's interstitial (which is a regular allocatable +DDT allocated once per thread at startup). The UFS interstitial is: +1. **Created** (`GFS_Interstitial(nt)%create(ixs, ixe, model)`) before each block — this + allocates all required fields to the chunk size `ixe-ixs+1` +2. **Reset** (`GFS_Interstitial(nt)%reset(model)`) to zero before radiation and phys_ps +3. **Destroyed** (`GFS_Interstitial(nt)%destroy(model)`) after each block — deallocates + +This design exists because different blocks (especially the last block) can have different +sizes. Pre-allocating to the maximum size wastes memory at scale; per-block allocation +ensures exact sizing. The pointer-based design also allows the `create()` method to +selectively allocate only the fields needed for the current physics configuration. + +In the caps, the interstitial is accessed as: +```fortran +gfs_interstitial(cdata%thrd_no)%del(chunk_begin:chunk_end, one:levs) +``` + +The interstitial array is 1-D (indexed by thread, not by `(instance, thread)` as in SCM). +This works because UFS has only one model instance at runtime — no ensemble-in-memory. + +--- + +### 11.9 Optional variables — the pointer array pattern at scale + +The phys_ps run cap has **200 optional pointer arrays** in its local variable section. +Each looks like: +```fortran +type :: real_kind_phys_rank1_ptr_arr_type + real(kind_phys), dimension(:), pointer :: p => null() +end type real_kind_phys_rank1_ptr_arr_type +type(real_kind_phys_rank1_ptr_arr_type), dimension(1:cdata%thrd_cnt) :: sfc_wts_1_ptr_array +``` + +Usage pattern (consistent with SCM but with threading dimension): +```fortran +if (gfs_control%lndp_type /= 0) then + sfc_wts_1_ptr_array(cdata%thrd_no)%p => & + gfs_coupling%sfc_wts(chunk_begin:chunk_end, one:gfs_control%n_var_lndp) +end if +! ... scheme call ... +if (gfs_control%lndp_type /= 0) then + nullify(sfc_wts_1_ptr_array(cdata%thrd_no)%p) +end if +``` + +The array is dimensioned by `cdata%thrd_cnt` (total thread count) and indexed by +`cdata%thrd_no` (current thread number). This handles the threaded run phase where +multiple threads are simultaneously executing the same run cap function with different +chunk ranges. Each thread independently associates and nullifies its own pointer slot. + +200 optional variables in `phys_ps` alone. This is the regime for which the SCM had ~550 +total optional vars — confirming that operational 3-D GFS physics is heavily optional-var +driven. The design is sound but generates enormous boilerplate. + +A key observation: the type definition for each pointer wrapper (`integer_..._ptr_arr_type`, +`real_kind_phys_rank1_ptr_arr_type`, etc.) is **re-declared inside every single function +that needs it**. This results in duplicate type definitions across all group caps. The +redesign should define these wrapper types once in a shared module. + +--- + +### 11.10 Physical constants as metadata variables + +The UFS static API has an extensive USE list of physical constants from `gfs_typedefs`: +``` +con_pi, con_g, con_t0c, con_hfus, con_solr_2008, con_solr_2002, con_c, con_plnk, +con_boltz, con_rd, ltp, con_zero, con_rerth, con_p0, con_rv, con_cp, con_rgas, +con_amd, con_amw, con_avgd, con_hvap, con_eps, con_omega, con_fvirt, con_ttp, +con_thgni, con_epsm1, con_rog, con_rocp, con_tice, con_sbc, con_jcal, con_rhw0, +rlapse, rhowater, karman, con_1ovg, con_cliq, con_cvap, rainmin, con_epsm1 (30+ total) +``` + +These travel through the full chain: static API USE → suite cap argument → group cap +argument → scheme call argument. Each constant is declared as a separate scalar dummy +argument (`real(kind_phys), intent(in), target :: con_pi`) in every group cap that needs +it. + +This is correct but verbose. The redesign should consider whether constants should be +gathered into a dedicated DDT (e.g., `gfs_constants_type`) so the cap chain carries one +argument instead of 30. This would also eliminate the need to explicitly enumerate which +constants each group needs — they could all come along in the constants DDT. + +--- + +### 11.11 The `one` lower-bound anchor + +The integer constant `one = 1` (from `ccpp_types`) is passed as an explicit argument +throughout the UFS cap chain for the same reason as in SCM: it anchors lower array bounds +without triggering association-status issues: +```fortran +type(gfs_interstitial_type), intent(inout), target :: gfs_interstitial(one:) +tgrs(one:gfs_control%ncols, one:gfs_control%levs) +``` + +This pattern is ubiquitous and is a known prebuild idiom. + +--- + +### 11.12 No framework-owned persistent variables + +Unlike CAM-SIMA (which allocates scheme-persistent variables in the suite cap), the UFS +has no framework-owned persistent state in any cap. All persistent state lives in the host +DDTs (`GFS_tbd`, `GFS_sfcprop`, etc.). The interstitial DDT (`GFS_interstitial`) is purely +transient — created and destroyed each block. + +This is consistent with UFS's prebuild-based architecture. Whether framework-owned +persistent variables would be beneficial for UFS is an open question for the redesign. + +--- + +### 11.13 Build system and driver + +Prebuild is invoked from CMake (not programmatically) and generates: +- Group cap files (one per group × suites) +- Suite cap files (one per suite) +- `ccpp_static_api.F90` +- `CCPP_CAPS.cmake`, `CCPP_SCHEMES.cmake`, `CCPP_TYPEDEFS.cmake` — consumed by CMake to + enumerate files to compile + +The host driver (`CCPP_driver.F90`) is **hand-written**, not auto-generated. It owns the +OpenMP loop, the cdata allocation/setup, the interstitial create/destroy, and the +diagnostic bucket zeroing. This is a significant difference from CAM-SIMA where the +equivalent driver code is partially generated. In the redesign, this host driver code +should remain hand-written — it encodes model-specific threading and blocking decisions +that cannot be derived from metadata alone. + +--- + +### 11.14 Observations relevant to the redesign + +1. **The DDT-argument cap chain is fully validated at UFS scale.** Passing 10+ DDTs plus + 30+ scalar constants as named arguments through three cap levels works correctly in + production. The redesign must replicate this exactly. + +2. **The chunk_begin/chunk_end subsetting at call sites is non-negotiable.** Hundreds of + array sections per group cap. The generator must produce this from the metadata + `horizontal_dimension` standard name and the `active` flag for optional variables. + This is prebuild's core value at 3-D scale. + + *Design direction*: Rather than carrying `chunk_no` in cdata and having the cap look + up `gfs_control%chunk_begin(chunk_no)`, the redesign should pass + `horizontal_loop_begin` and `horizontal_loop_end` as explicit arguments directly to + `ccpp_physics_run()` (and analogous calls). This decouples the cap from knowing about + the host's internal chunk-lookup arrays. The host driver sets these for each block + iteration and passes them in; the cap uses them directly. + +3. **The domain-vs-block execution contexts must be supported, but the cdata object is + not necessarily the right mechanism.** The key information is: instance number, thread + number, horizontal_loop_begin, horizontal_loop_end, error flag/message. If all of + these are explicit named arguments to `ccpp_physics_*`, the cdata object becomes + redundant scaffolding. This is an open design question to be discussed separately, but + the UFS analysis shows that cdata carries exactly these values — the object is a + transport container, not a framework abstraction. + +4. **The `blksz` non-uniform block size is a first-class concern.** The generator must + produce `im = gfs_control%blksz(cdata%blk_no)` (or an equivalent `horizontal_loop_extent` + computed from the explicit begin/end) for the horizontal extent argument in run phases. + +5. **GFS_interstitial as a pointer-DDT is the correct design for 3-D models.** Creating + and destroying per block avoids memory waste from over-allocation to the maximum chunk + size. The pointer-based field design enables selective allocation. The redesign should + document this pattern and support it. (Whether the generator should emit the + `type(X_interstitial_type)` DDT definition itself or only the caps is TBD.) + +6. **200 optional pointer arrays in one group cap is manageable but the wrapper type + proliferation is not.** The 4 wrapper types (`integer_r1_ptr_arr_type`, + `real_r1_ptr_arr_type`, `real_r2_ptr_arr_type`, `character_len3_r1_ptr_arr_type`) + should be defined once in a shared module (e.g., `ccpp_types.F90`) and reused across + all caps, eliminating thousands of duplicate lines. + +7. **Physical constants as metadata variables must be gathered into a constants DDT.** + The redesign will collect all physics constants into a single `constants_type` DDT + (or equivalent), reducing 30+ individual scalar arguments in the cap chain to one + argument. This requires a metadata declaration mechanism for compound read-only + objects (i.e., constants do not need intent tracking the way state variables do). + +8. **No framework-owned persistent variables in UFS** confirms that this feature is + optional and model-specific. The redesign needs to support it (for CAM-SIMA-like + models) but should not force it on models that do not need it. + +9. **The host driver is correctly hand-written.** The OpenMP blocking, interstitial + lifecycle, diagnostic bucket management — these are model-specific decisions that + belong in the host driver, not in generated code. The redesign should not try to + generate the driver. + +10. **Suite variant cap redundancy is not a concern.** For research/development, multiple + suites are active simultaneously and generated code size doesn't matter. For + production, only one suite is compiled and used at a time. The redesign need not + prioritize eliminating redundant group cap code across suite variants. + +--- + +## 12. Real-world example: Navy NEPTUNE (prebuild, restricted) + +The NEPTUNE source code cannot be shared. The following is based on architectural +description provided by the lead developer. + +NEPTUNE uses `ccpp-prebuild` with the same GFS physics as UFS and nearly identical suites. +Its unique distinguishing feature is **multiple coexisting CCPP physics instances** — it +is the only model among the four examples that exercises this capability at runtime. + +--- + +### 12.1 Multiple instances — the N-dimensioned DDT array mechanism + +In NEPTUNE, the host model allocates N copies of all GFS DDTs as 1-D arrays indexed by +instance number: + +```fortran +type(GFS_sfcprop_type), allocatable :: gfs_sfcprop(1:N) +type(GFS_statein_type), allocatable :: gfs_statein(1:N) +type(GFS_stateout_type), allocatable :: gfs_stateout(1:N) +! ... all GFS DDTs dimensioned 1:N +type(GFS_control_type), allocatable :: gfs_control(1:N) +``` + +The static API imports these module-level arrays via `use` statements (same as UFS). +The instance selection happens at the call site inside the group cap, using +`cdata%ccpp_instance` as the array index: + +```fortran +call foo_run( & + tair = gfs_statein(cdata%ccpp_instance)%tair( & + gfs_control(cdata%ccpp_instance)%chunk_begin(cdata%chunk_no) : & + gfs_control(cdata%ccpp_instance)%chunk_end(cdata%chunk_no), & + 1:nvertical), & + ...) +``` + +Three things are happening simultaneously at each call-site array section: +1. **Instance selection**: `gfs_statein(cdata%ccpp_instance)` picks the correct DDT from + the N-element array +2. **Chunk subsetting**: `chunk_begin(chunk_no):chunk_end(chunk_no)` applies the run-phase + horizontal slice +3. **Vertical bound**: explicit `1:nvertical` + +This is the same pattern as UFS except the DDTs are 1-D arrays rather than scalars. +The generator must produce this instance-indexed subsetting when the host declares its +DDTs as arrays. + +--- + +### 12.2 What NEPTUNE tells us about `cdata%ccpp_instance` + +The `initialized(200)` array in every group cap (confirmed in both SCM and UFS caps) now +has its full motivation: it handles up to 200 simultaneous instances without requiring +per-instance cap code. The `cdata%ccpp_instance` value (1-based) is the runtime selector +into both the host DDT arrays and the `initialized` guard array. + +NEPTUNE is the reason `200` is not `1`. In single-instance models (UFS, SCM, CAM-SIMA) +`cdata%ccpp_instance` is always 1 and the N-dimensioned DDT arrays have `N=1`. + +--- + +### 12.3 Observations relevant to the redesign + +1. **Multiple instances require only one change at the call site**: inserting the instance + index at the correct dimension position. Everything else (chunking, optional variables, + threading) composes with this unchanged. + +2. **The instance dimension can appear anywhere in any host variable — not just as an + index into an array of DDTs.** A flat array `flat_field(1:ninstance, 1:nhoriz, 1:nvert)` + is equally valid; its call site becomes: + ```fortran + flat_field(instance_number, horiz_begin:horiz_end, 1:nvert) + ``` + The generator handles this by classifying each dimension by its declared standard name. + `instance_dimension` is a registered standard name (like `horizontal_dimension` and + `vertical_dimension`) — the generator knows its semantics regardless of where it + appears in the dimension list or whether the variable is a DDT array element or a + plain array. See §13.4 for the full dimension classification model. + +3. **No new cap-level mechanism is needed for multi-instance.** The instance number + (from the control layer, see §13) is sufficient. The cap code shape is the same; + only the call-site indexing expression differs based on the declared dimension roles. + +--- + +## 13. Cross-cutting design decision: how host data enters the cap chain + +Across all four models, two mechanisms are used for getting host model data into the +generated caps: + +| Mechanism | Models using it | Description | +|-----------|----------------|-------------| +| **Module USE** | UFS, SCM, CAM-SIMA, NEPTUNE | Static API has `use ccpp_data, only: gfs_statein, ...`. Data module name is known at generation time. | +| **Command-line arguments** | capgen (optional) | Generator accepts host variable access paths as CLI flags; generated caps receive data as explicit dummy arguments. | + +### 13.1 The capgen dual-mechanism problem + +Capgen supports both mechanisms, and this is a direct source of its complexity. The +variable-matching logic, VarDictionary scope chains, and `CCPPDatabaseObj` all exist +partly to handle the routing of variables that may arrive via either path. Maintaining +two entry points to the data layer doubles the surface area that must be tested and +reasoned about. + +### 13.2 The proposed single-mechanism approach + +The redesign will use **module USE exclusively** for all host data. The reasoning: + +- All four production models already use module USE, including CAM-SIMA (the capgen + model), which does not use capgen's CLI-argument path in practice. +- Module names are stable, known at generation time, and make the generated code + self-documenting (`use ccpp_data, only: gfs_statein` is unambiguous). +- Eliminating the CLI-argument entry path eliminates an entire class of generator + complexity. + +### 13.3 Runtime control variables — the thin explicit layer + +While all *data* enters via module USE, a set of *control* variables must be passed at +runtime because they change from call to call. These are not physics data; they tell the +cap *how* to index into the data it already has access to: + +| Variable | Purpose | When it matters | +|----------|---------|----------------| +| `ccpp_instance` | Select the instance dimension in host variables | NEPTUNE (N>1); others use 1 | +| `ccpp_thread_no` | Index optional pointer arrays per thread | Run phase with OpenMP | +| `horizontal_loop_begin` | Start of horizontal chunk to process | Run phase | +| `horizontal_loop_end` | End of horizontal chunk to process | Run phase | +| `ccpp_nthreads` | Max threads available for internal physics use | Non-run phases (currently `gfs_control%nthreads`) | +| `errmsg` / `errflg` | Error reporting return path | All phases | + +These are exactly the values that `cdata` carries in the current implementation. +Whether they are packaged as a `ccpp_t` struct or passed as individual named arguments to +`ccpp_physics_*` is an open design question for implementation. Either way, the generator +only needs to know about these variables and their standard names — it does not need to +accept host data paths on the command line. + +### 13.4 The dimension classification model + +A host variable's metadata declares the **standard name of each of its dimensions** in +order. The generator classifies every dimension into one of three categories and +constructs the call-site expression accordingly. + +**Category 1 — Registered dimensions.** The generator knows the semantics of these +standard names and generates special call-site expressions for them: + +| Standard name | Call-site expression | Notes | +|--------------|---------------------|-------| +| `instance_dimension` | `instance_number` (scalar index) | Omitted if variable has no instance dimension | +| `horizontal_dimension` | `1:horizontal_dimension` (non-run) or `horiz_begin:horiz_end` (run) | Phase-dependent | +| `vertical_dimension` | `1:vertical_dimension` | Fixed range | + +`instance_dimension` has the same registered status as `horizontal_dimension` and +`vertical_dimension`. Single-instance models simply do not declare any variables with +an `instance_dimension`, and the generator omits that index entirely. + +**Category 2 — Arbitrary host-declared dimensions.** Any dimension whose standard name +is not in the registered set. These are declared in host metadata pointing to a Fortran +expression accessible via module USE — either a flat module variable or a DDT member +(e.g. `gfs_control%ntrac`, `gfs_control%kice`). The generator emits `1:expression` +at the call site, resolved at generation time from the metadata. Fixed-index extractions +(e.g. `gfs_statein%qgrs(..., gfs_control%ntqv)`) are a special case: the dimension +value is a scalar index rather than a range upper bound, and the metadata must declare +which case applies. + +**Category 3 — Optional selector.** Not a dimension per se, but a boolean `active` +condition declared in variable metadata. Generates a pointer-association guard around +the call site (the pattern described in §9 and §11). + +This three-category model works uniformly regardless of host layout: +- `gfs_statein(instance)%tair(horiz, vert)` — registered instance + registered horizontal + registered vertical +- `flat_field(instance, horiz, vert, ntrac)` — registered + registered + registered + arbitrary +- `flat_field(horiz, vert)` — no instance dimension, single-instance model + +No special-casing per host model is needed in the generator. + +### 13.5 `type = control` — metadata declaration for runtime control variables + +The registered dimensions (§13.4 Category 1) are *dimension names* that appear in a +variable's `dimensions = (...)` list. Their actual *runtime values* are supplied by a +separate set of variables declared with `type = control` in host metadata. + +| `type = control` standard name | Fills in registered dimension / purpose | +|-------------------------------|----------------------------------------| +| `ccpp_instance` | `instance_dimension` — scalar index selecting the active instance | +| `ccpp_thread_no` | Not a dimension; indexes optional pointer arrays per thread | +| `horizontal_loop_begin` | Lower bound of `horizontal_dimension` in run phase | +| `horizontal_loop_end` | Upper bound of `horizontal_dimension` in run phase | +| `ccpp_nthreads` | Not a dimension; max threads available for internal physics use | +| `errmsg` / `errflg` | Error reporting return path | + +Variables declared `type = control` are: +- **Passed explicitly as runtime arguments** to `ccpp_physics_*` by the host driver + (not accessed via module USE, because their values change per call) +- **Used by the generator** to construct call-site indexing expressions for registered + dimensions, and to generate the `ccpp_nthreads` assignment before non-run scheme calls +- **Available to physics schemes** by standard name like any other variable — if a scheme + declares a variable with a matching standard name (e.g. `ccpp_nthreads`, + `horizontal_loop_begin`), the framework passes it as a scheme argument in the normal way + +This is similar in concept to capgen's `type = host` annotation but with a narrower, +well-defined scope. The name `control` is intentional: these variables *control* how +the cap indexes into the data, not what the data is. + +The set of recognized standard names for `type = control` variables is fixed and small. +Declaring them explicitly in metadata — rather than having the generator recognize magic +names — keeps the mechanism open and self-documenting. + +### 13.6 Consequences for the generator + +1. The generator reads host metadata to learn: + - Module names for all host data variables (emitted as `use` statements in the static API) + - The dimension standard names of each variable (for call-site expression construction) + - Which variables are `type = control` (for the runtime argument layer) +2. At cap generation time, the static API's `use` statements are emitted from the module + names — no runtime flexibility, no CLI data routing. +3. Call-site subsetting for every variable is constructed purely from its declared + dimension standard names: registered dimensions use the Category 1 rules; arbitrary + dimensions are resolved to Fortran expressions via the host metadata. +4. The only runtime inputs to the cap are the `type = control` variables. Their values + are supplied by the host driver for each `ccpp_physics_*` call. diff --git a/doc/redesign_prompt.md b/doc/redesign_prompt.md new file mode 100644 index 00000000..5ed79126 --- /dev/null +++ b/doc/redesign_prompt.md @@ -0,0 +1,1438 @@ +# CCPP Framework Code Generator — Redesign Specification + +*Last revised: 2026-05-13 (late evening — SCM-driven session).* + +## Purpose + +This document is a complete implementation specification for a new CCPP Framework code +generator (`ccpp-capgen`). It supersedes both `ccpp-prebuild` and `ccpp-capgen`. An +implementer should be able to build the new generator from scratch using this document +alone, supplemented by the real-world examples in `redesign_analysis.md`. + +The spec is essentially as-implemented as of the date above. User-facing +deltas relative to ccpp-prebuild and the original ccpp-capgen are +collected in `doc/migration.md`; section 18 of this document is a rolling +"outstanding work" tracker. + +--- + +## 1. Background and Motivation + +The CCPP Framework couples host NWP models (UFS Weather Model, NEPTUNE, CCPP-SCM, +CAM-SIMA) to physics parameterization schemes by auto-generating Fortran interface +("cap") code. Two generators exist today: + +- **`ccpp-prebuild`** — simple, procedural Python; fast; DDT arguments; used in + production by UFS, NEPTUNE, SCM. Does not support framework-owned variables. +- **`ccpp-capgen`** — complex OO Python; flat-field arguments; used in CAM-SIMA. The + deep class hierarchy (`VarDictionary`, `VarCompatObj`, `CCPPDatabaseObj`, etc.) makes + it unmaintainable. Three developers spent considerable time trying to add DDT argument + passing and could not succeed. + +The redesign starts fresh, drawing lessons from both. The guiding principle is: +**simplicity of prebuild, feature set of capgen**. + +The primary failures that triggered the redesign: +1. capgen passes flat fields to group caps — infeasible at UFS/NEPTUNE scale (1200+ + variables), breaks under compiler debug flags for optional variables. +2. capgen's scope-chain variable promotion is the source of most complexity. +3. Nobody on the team fully understands capgen. + +--- + +## 2. Toolchain Structure + +The redesign produces **two separate tools** that share the same metadata parsing library: + +### 2.1 Validator (`ccpp_validator.py`) + +Parses both Fortran source files and metadata files, compares them, and reports +discrepancies. Run by developers before invoking the generator — e.g., during scheme +development or in CI. Does **not** generate any Fortran output. + +For each scheme phase declared in a `.meta` file, the validator checks that the +corresponding Fortran subroutine: (1) exists in the source tree, (2) has the same +number of dummy arguments, (3) the argument names match the `local_name` values in +the metadata (order-insensitive), and (4) for every argument present in both sides, +the `intent`, `type`, `kind`, and dimension *rank* agree. `character` arguments +treat `len=*` on either side as a wildcard. DDT names compare against the Fortran +`type(name)` / `class(name)` wrapper; `external::` metadata +compares against the Fortran `type(typename)` (the module qualifier is +metadata-only). + +The `optional` attribute is asymmetric: metadata `optional=True` paired with a +Fortran dummy that is *not* declared `optional` is a hard error (the cap's +`present()` check would be invalid on a Fortran-required dummy); the reverse +direction (Fortran-only `optional`) is a warning, since always passing the arg is +a valid subset of the Fortran contract. + +Fortran source files can be supplied explicitly on the CLI (`--source-files`). When +omitted, the validator auto-discovers the Fortran source for each scheme table using the +`source_path` table-level property (Section 3.5): it looks for a `.F90` file with the +same base name as the `.meta` file, in the directory given by `source_path`. + +### 2.2 Code Generator (`ccpp_capgen.py`) + +Parses metadata only. Assumes metadata correctly describes the Fortran source — performs +no Fortran parsing. Generates all cap files and supporting modules. + +**Both tools import the same metadata parsing module.** No duplication of metadata +parsing logic between the two tools. + +--- + +## 3. Metadata Format + +### 3.1 File format + +The existing ini-file format is preserved unchanged. Every metadata file consists of +`[ccpp-table-properties]` header blocks followed by `[ccpp-arg-table]` variable listing +blocks, exactly as in the current framework. + +The `[ccpp-table-properties]` + `[ccpp-arg-table]` pair is redundant for non-scheme +tables (the distinction is vestigial for host/DDT/suite tables) but is preserved for +symmetry with scheme metadata tables. + +### 3.2 Table types (`type =` in `[ccpp-table-properties]`) + +Five table types are supported: + +| `type =` | Ownership | Import mechanism | +|---|---|---| +| `scheme` | Physics scheme | Intent args on scheme subroutines | +| `host` | Host model | Module USE (direct or via DDT member) | +| `control` | Framework runtime layer | Explicit args to `ccpp_physics_*` entry points | +| `suite` | Generated suite cap | Module USE of generated suite data module | +| `ddt` | Type definition | Structural — describes DDT fields, no instance info | + +Notes: +- `type = module` from capgen is renamed to `type = host`. Breaking change, intentional. +- `type = suite` tables are **written by the generator** (never hand-authored). They + appear on disk for inspection and debugging only. +- `type = ddt` describes the structure of a Fortran derived type. It contains no + instance information — only field definitions. + +### 3.3 Per-variable attributes + +All existing per-variable attributes are preserved: `standard_name`, `long_name`, +`units`, `dimensions`, `type`, `kind`, `intent`, `optional`, `active`, `protected`. + +`protected = True` means: any scheme that declares `intent` other than `in` for this +variable is a metadata error, caught at generation time. This is how constants are +handled — a constants DDT is declared `type = host` with all fields `protected = True`. +No separate `type = constants` is needed. + +### 3.4 DDT type definitions + +A DDT type definition uses `type = ddt`: + +```ini +[ccpp-table-properties] + name = gfs_statein_type + type = ddt + +[ccpp-arg-table] + name = gfs_statein_type + type = ddt + +[phii] + standard_name = geopotential_at_interface + long_name = geopotential at model layer interfaces + units = m2 s-2 + dimensions = (horizontal_dimension, vertical_interface_dimension) + type = real + kind = kind_phys +``` + +### 3.5 Table-level properties + +The `[ccpp-table-properties]` block supports the following table-level keys beyond `name` +and `type`: + +| Key | Applies to | Purpose | +|---|---|---| +| `source_path` | `scheme` | Relative path from the `.meta` file directory to the directory containing the corresponding Fortran `.F90` source file. Defaults to the `.meta` file's own directory if absent. Used by the validator for auto-discovery of Fortran source. | +| `dependencies` | `scheme`, `host` | Comma-separated list of dependency file names or relative paths. Resolved to absolute paths using `dependencies_path` as a base directory (or the `.meta` file's directory if `dependencies_path` is absent). | +| `dependencies_path` | `scheme`, `host` | Optional subdirectory (relative to the `.meta` file's directory) used as the base when resolving entries in `dependencies`. Has no effect if `dependencies` is absent or `none`. | + +Example: + +```ini +[ccpp-table-properties] + name = my_scheme + type = scheme + source_path = ../src + dependencies_path = ../deps + dependencies = utility_module.F90, shared_constants.F90 +``` + +The resolved `dependencies` paths are collected across all scheme tables and written to the +`` section of `datatable.xml`. The validator uses `source_path` (not +`dependencies`) for locating the Fortran `.F90` corresponding to each `.meta` file. + +**Parser implementation note:** The INI parser applies these table-level properties to +the `MetadataTable` object before transitioning to any `[ccpp-arg-table]` section. A +`flush_table_props()` call must happen at every parser-state transition (new table +header, first arg-table header, end-of-file) to avoid silently discarding the properties. + +### 3.6 DDT instances + +A DDT instance is declared as a regular variable entry inside a `type = host` table. +The enclosing `[ccpp-table-properties]` block's `name` attribute identifies the Fortran +module from which the instance is imported via `use`. No separate `module` attribute is +needed on the variable entry. + +```ini +[ccpp-table-properties] + name = CCPP_data + type = host + dependencies = CCPP_typedefs.F90,GFS_typedefs.F90 + +[ccpp-arg-table] + name = CCPP_data + type = host + +[gfs_statein] + standard_name = gfs_statein + long_name = GFS state input for all instances + units = mixed + dimensions = (number_of_instances) + type = gfs_statein_type +``` + +The generator looks up the `gfs_statein_type` DDT table, traverses its fields, and +constructs access paths of the form +`gfs_statein(instance_number)%fieldname(loop_begin:loop_end, 1:nlevs)`. + +For scalar DDT instances (no instance dimension), dimensions is `()`. +For nested DDTs, the same mechanism applies recursively. + +### 3.7 Control variable declarations + +Control variables are declared in a `type = control` table. The generator resolves +each control variable by its standard name and uses whatever local Fortran name the +host declared: + +```ini +[ccpp-table-properties] + name = my_host_control_module + type = control + +[ccpp-arg-table] + name = my_host_control_module + type = control + +[loop_begin] + standard_name = horizontal_loop_begin + long_name = start of horizontal loop + units = index + dimensions = () + type = integer +``` + +--- + +## 4. Control Variables + +The generator recognizes the following standard names for control variables. Local +Fortran names are host-defined (resolved from the `type = control` metadata table). + +### 4.1 Entry point arguments (non-register phases) + +All required control variables are unconditional — every host must declare all of them. +Models that don't use a variable pass the neutral value: `1` for integers, `''` for +character arguments. + +| Standard name | Expected type | Role | +|---|---|---| +| `suite_name` | `character` | Suite name for runtime dispatch | +| `horizontal_loop_begin` | `integer` | Start of horizontal slice (chunk bounds for `ccpp_physics_run`; `1` for all other phases) | +| `horizontal_loop_end` | `integer` | End of horizontal slice (chunk bounds for `ccpp_physics_run`; `ncols` for all other phases) | +| `number_of_physics_threads` | `integer` | Thread budget for physics-internal OpenMP; pass `1` if none | +| `ccpp_error_message` | `character` | Error message string | +| `ccpp_error_code` | `integer` | Integer error return code | + +**Two symmetric paired-optional `(index, count)` control pairs** — +`instance_number` / `number_of_instances` and `thread_number` / +`number_of_threads`. For each pair, declare **both** members in +`type=control` for the multi-instance / multi-threading API, or +**neither** for the single API; declaring exactly one is a hard error. +When a pair is absent, the static API drops the index argument and the +framework uses literal `1` where it would appear (and, for instances, +per-instance state arrays size to 1). A host variable may be +dimensioned by a count standard name only when its pair is declared — +the `SCALAR_INDEX_DIMS` collapse substitutes the index local name +(`instance_number` / `thread_number`) and errors if it isn't in scope. + +The asymmetry between the pairs is in *who reads the count*, not in the +rules: the framework reads `number_of_instances` to size its own +per-instance state; it does not yet read `number_of_threads` +(per-thread containers are host-owned) but carries it for future +symmetry. + +`group_name` is **not** in the required set. It is included in the static API signature +only if the host declares it in their `type=control` table. When absent: the static API +calls all groups in declared order; no dispatch argument is generated; the generator +warns (not errors) if any loaded suite has more than one group. When present: it is a +required (non-optional) `character` argument; the value `''` (empty string) or `'all'` +calls all groups in order; any other value dispatches to the named group only. + +The generator validates the required set at startup (after host metadata is parsed): +every required standard name must be present with the expected Fortran type (rank-0 +scalar). All failures are collected and reported together before halting. + +### 4.2 Loop-generated control variables (subcycles only) + +| Standard name | Role | +|---|---| +| `ccpp_loop_counter` | Current subcycle iteration (1..ccpp_loop_extent) | +| `ccpp_loop_extent` | Total subcycle iterations; value comes from the `loop=N` attribute on the `` element in the suite XML definition file | + +These are **not** passed as `ccpp_physics_*` arguments from the host. They are set by +the generated `do` loop inside the group cap and are available to any scheme called +within that loop. Outside a subcycle loop, these variables are not in scope. + +### 4.3 Registered dimension standard names + +The generator has built-in semantic knowledge of these dimension standard names: + +| Standard name | Indexing semantic | +|---|---| +| Any key of `SCALAR_INDEX_DIMS` (currently `number_of_instances`, `number_of_threads`) | Scalar extraction: substitute the paired index variable's local Fortran name (currently `instance_number`, `thread_number`). See `capgen/metadata/registered_dimensions.py` for the full table and the contract. | +| `horizontal_dimension` | **At scheme call sites**: always `horizontal_loop_begin:horizontal_loop_end` (using control variable local names), for all phases. **For suite-owned array allocation sizing**: local name of `horizontal_dimension` from the host `type=host` table (accessed via module USE, not the control variable). | +| `vertical_*` | Slice: `1:` | + +`horizontal_dimension` and `vertical_*` are "registered" because the generator knows +their slicing semantics, but they are resolved to local names the same way as arbitrary +dimensions — by looking up the variable with that standard name in the host metadata. +**Scheme metadata always uses `horizontal_dimension`** for the horizontal extent, regardless of which phase the entrypoint belongs to. The standard name `horizontal_loop_extent` does not exist in the new design. The distinction between a chunk call and a full-domain call is handled entirely by what the host passes for `horizontal_loop_begin` and `horizontal_loop_end` — invisible to scheme developers and to the cap generator's slicing logic. + +All other dimension standard names are resolved identically: look up the variable with +that standard name, get its local Fortran name, emit `1:local_name`. + +Registered scalar-index dims are subject to one hard contract (Rule 2 in +`metadata/registered_dimensions.py`): they may appear only on **container +DDT-instance variables** in the access path, never on leaf data variables +(intrinsic- or `external:`-typed). Leaves that declare them are rejected +at parse time with a remediation pointer. See `doc/migration.md` §3.4. + +--- + +## 5. Entry Points + +Eight entry points are generated in the static API. Two tiers: + +### 5.1 Framework lifecycle (no group_name dispatch) + +These operate on the entire suite at once. They take `suite_name`, +`ccpp_error_code`, and `ccpp_error_message` (plus `instance_number` when the +host opts into the multi-instance pair, §4.1). No scheme `_run/_init/_final` +calls. + +| Entry point | Purpose | +|---|---| +| `ccpp_register(suite_name, errcode, errmsg, [instance_number])` | Calls each scheme's `_register` entrypoint; transitions suite state to `REGISTERED`. Auto-provisions `ccpp_model_constituents_obj(:)` and friends in `ccpp_host_constituents.F90` when any register-phase scheme declares `ccpp_constituent_properties_t(:)` (constituents are not a formal arg). | +| `ccpp_init(suite_name, errcode, errmsg, [instance_number])` | Allocates integer state arrays in all group caps; allocates suite-owned interstitial data; calls the suite-level `` scheme if declared (§5.5); no per-group scheme calls. | +| `ccpp_final(suite_name, errcode, errmsg, [instance_number])` | Calls the suite-level `` scheme if declared (§5.5); deallocates integer state arrays and suite-owned data; no per-group scheme calls. | + +Constituents are **opt-in**: a separate generated module +`ccpp_host_constituents.F90` declares `ccpp_model_constituents_obj(:)` and a +host-facing API (`ccpp_register_constituents`, `ccpp_initialize_constituents`, +`ccpp_const_get_index`, `ccpp_constituents_array(instance_number)`, +`ccpp_advected_constituents_array`, `ccpp_model_const_properties`, +`ccpp_number_constituents`, `ccpp_gather_constituents`, +`ccpp_update_constituents`, `ccpp_is_scheme_constituent`). The host calls +these directly — they are not formal arguments of `ccpp_register` / `ccpp_init`. +See `doc/constituents.md`. + +`instance_number` appears in every framework-lifecycle signature only when +the host declares the `instance_number` / `number_of_instances` pair (§4.1). +When present it propagates: `ccpp_init` → `_init` → each group's +`state_alloc(number_of_instances, ...)`. + +### 5.2 Physics group invocation (dispatched by suite_name + group_name) + +| Entry point | Calls scheme phase | +|---|---| +| `ccpp_physics_init(...)` | `_init` | +| `ccpp_physics_timestep_init(...)` | `_timestep_init` | +| `ccpp_physics_run(...)` | `_run` | +| `ccpp_physics_timestep_final(...)` | `_timestep_final` | +| `ccpp_physics_final(...)` | `_final` | + +All five take the full required control variable argument list (Section 4.1) — a uniform +signature across all phases. If `group_name` is declared in the host's `type=control` +table, it is also included; `''` or `'all'` calls all groups in order, any other value +dispatches to the named group only. If `group_name` is absent from the control table, +no dispatch argument is generated and all groups are called in order. + +The host is responsible for passing appropriate horizontal bounds: actual chunk bounds +for `ccpp_physics_run`; `1` and `ncols` (full domain) for all other phases. The cap +always uses `(horizontal_loop_begin:horizontal_loop_end)` for array slices — no +phase-specific special-casing. + +### 5.3 Naming note + +`finalize` is renamed to `final` throughout (e.g., `ccpp_physics_final`, not +`ccpp_physics_finalize`). Breaking change, intentional for symmetry: +`ccpp_init`/`ccpp_final`, `ccpp_physics_init`/`ccpp_physics_final`, +`ccpp_physics_timestep_init`/`ccpp_physics_timestep_final`. + +The SDF likewise accepts only the canonical short element names: +`` and `` (§5.5). The legacy long spellings — `` +(old typo), `` (correct long form), `` — are +rejected at parse time with a clear error pointing at the short form. + +### 5.5 Suite-level lifecycle hooks (`` / ``) + +The SDF root may declare a **single** scheme that runs at suite-init +and/or suite-final time: + +```xml + + my_init_scheme + … + my_final_scheme + +``` + +The named scheme's `init` (resp. `final`) phase is resolved from the +scheme metadata and called from inside `_init` (resp. +`_final`). Ordering: + +- `_init`: after all group `state_alloc` and + `suite_data_init_fields`, **before** the `CCPP_SUITE_FRAMEWORK_INITIALIZED` + state transition. An errflg from the init scheme prevents the + state transition. +- `_final`: before the `CCPP_SUITE_UNREGISTERED` transition. + +Constraints: + +- One scheme per `` / ``. Multiple `` children + inside (the "group" shape) is a schema violation. +- The named scheme must have the matching phase in its metadata. + Missing-phase metadata is a generator error. + +### 5.4 Suite introspection routines + +In addition to the eight entry points above, the static API exposes **five** +suite-introspection subroutines that let a host query, at runtime, what is +compiled into the API. These mirror the equivalent routines in the original +capgen (`scripts/ccpp_suite.py` — `write_inspection_routines`) and are +used by CMake integration and host-side build glue. + +| Entry point | Purpose | +|---|---| +| `ccpp_physics_suite_list(suites)` | Return all suite names compiled into the API | +| `ccpp_physics_suite_part_list(suite_name, part_list, errmsg, errflg)` | Return the list of group ("part") names for a given suite | +| `ccpp_physics_suite_schemes(suite_name, scheme_list, errmsg, errflg)` | Return the list of scheme module names that compose a suite | +| `ccpp_physics_suite_variables(suite_name, variable_list, errmsg, errflg, [input_vars], [output_vars], [struct_elements])` | Standard-name list a suite consumes/produces; optional flags filter by intent and whether DDT sub-fields are flattened | +| `ccpp_physics_suite_host_data(suite_name, variable_list, errmsg, errflg)` | Standard-name list of host data the suite reads — DDT-collapsed view, excludes generated control variables | + +These routines do not advance the state machine and do not call any scheme +entrypoints. All inputs derive from generator-time data already held in +`SuiteResolution` plus the host/scheme metadata; no new metadata is required. +The `_variables` vs `_host_data` split distinguishes the flat-leaf view +(every DDT field that is actually consumed) from the DDT-collapsed view +(parent DDT instances), and excludes capgen-generated control +variables from `_host_data` since the host owns those. + +--- + +## 6. Cap Hierarchy + +All three levels are fully auto-generated. No hand-written components in the cap layer. + +### 6.1 Static API (`_ccpp_cap.F90`) + +- Imports all host DDTs and flat fields via `module use` (resolved from host metadata) +- Does not USE `ccpp_kinds` directly: the static API has no kind-typed declarations of + its own (it dispatches by `suite_name` and forwards control args). `ccpp_kinds` is + USEd only by files that declare kind-typed variables: group caps, the suite types + module, and the suite data module. +- Dispatches all eight entry points by `suite_name` to the appropriate suite cap +- Does not own constituent state; constituents are accessed via the separate + `ccpp_host_constituents.F90` module by both the host and group caps +- Holds no physics state + +### 6.2 Suite cap (`ccpp__cap.F90`) + +- Imports the generated suite data module (`ccpp__data.F90`) +- Contains the suite-level integer state array: `integer, allocatable :: ccpp_suite_state(:)` indexed by instance +- Implements the suite-level state machine (see Section 7) +- On `ccpp_register`: calls all scheme `_register` entrypoints across the suite +- On `_init(number_of_instances, errmsg, errflg)`: calls `state_alloc` for every + group, passing `number_of_instances` (or literal `1` for single-instance hosts); + also allocates suite-owned interstitial data. The `number_of_instances` argument is + conditional on the host declaring it (Section 7.2.1). +- On `ccpp_final`: deallocates all of the above +- Routes `ccpp_physics_*` calls by `group_name` to the appropriate group cap function; + passes `instance_number` (if present) through to group cap `_init` and `_final` subs + +### 6.3 Group cap (`ccpp___cap.F90`) + +- Imports `ccpp__data` (suite-owned interstitial data) +- Imports `ccpp__types` (shared wrapper types) +- Contains the group-level integer state array: `integer, allocatable :: ccpp_group_state(:)` indexed by instance +- Implements the group-level state machine (see Section 7) +- Contains the actual scheme call sites for each phase: + - Loop bound locals + - Optional variable pointer arrays (thread-dimensioned) + - Fixed-index extraction locals + - Unit/kind conversion locals + - Subcycle `do` loops + - Scheme calls with full argument lists + +--- + +## 7. State Machine + +Integer state parameters are defined as **private named parameters directly inside each +generated group cap module** — they are NOT imported from a shared framework library +module. Each group cap file declares: + +```fortran +integer, parameter, private :: CCPP_GROUP_UNINITIALIZED = 0 +integer, parameter, private :: CCPP_GROUP_INITIALIZED = 1 +integer, parameter, private :: CCPP_GROUP_IN_TIMESTEP = 2 +``` + +This means the integer values are replicated across generated files (acceptable — the +names are the contract, not the values). No generated file USEs a framework state module. + +Two levels, both indexed by `instance_number`. + +### 7.1 Suite-level state (in suite cap) + +```fortran +integer, parameter :: CCPP_SUITE_UNREGISTERED = 0 +integer, parameter :: CCPP_SUITE_REGISTERED = 1 +integer, parameter :: CCPP_SUITE_FRAMEWORK_INITIALIZED = 2 +``` + +| Entry point | Required state | State after | +|---|---|---| +| `ccpp_register` | `== UNREGISTERED` | `REGISTERED` | +| `ccpp_init` | `== REGISTERED` | `FRAMEWORK_INITIALIZED` | +| `ccpp_physics_*` (non-final) | `== FRAMEWORK_INITIALIZED` | (unchanged) | +| `ccpp_physics_final` | (idempotent: silent skip if state array unallocated or `== UNREGISTERED`); otherwise `== FRAMEWORK_INITIALIZED` | (unchanged) | +| `ccpp_final` | (idempotent: silent skip if state array unallocated or `== UNREGISTERED`); otherwise any `>= REGISTERED` | `UNREGISTERED` (state array deallocated on last-to-leave) | + +### 7.2 Group-level state (in each group cap) + +```fortran +integer, parameter :: CCPP_GROUP_UNINITIALIZED = 0 +integer, parameter :: CCPP_GROUP_INITIALIZED = 1 +integer, parameter :: CCPP_GROUP_IN_TIMESTEP = 2 +``` + +| Entry point | Required state | State after | +|---|---|---| +| `ccpp_physics_init` | `< INITIALIZED` (idempotent silent skip if `== INITIALIZED`) | `INITIALIZED` | +| `ccpp_physics_timestep_init` | `== INITIALIZED` | `IN_TIMESTEP` | +| `ccpp_physics_run` | `== IN_TIMESTEP` | `IN_TIMESTEP` | +| `ccpp_physics_timestep_final` | `== IN_TIMESTEP` | `INITIALIZED` | +| `ccpp_physics_final` | `>= INITIALIZED` (idempotent silent skip if `== UNINITIALIZED`) | `UNINITIALIZED` | + +The idempotency rule for `ccpp_physics_init`: if the group is already in state +`INITIALIZED`, return immediately without calling any scheme `_init` routines. This +allows the host to call `ccpp_physics_init` multiple times safely. Any further call +after the first must result in no change (idempotency is a scheme contract). + +The same rule applies to `ccpp_physics_final`: a repeat call (or a call issued +after `ccpp_final` has torn the suite down) must return cleanly with `errflg=0` +rather than erroring. This is enforced at both levels — the suite-cap dispatcher +silent-returns when `ccpp_suite_state` is unallocated or `== UNREGISTERED`, and +the group cap silent-returns when `ccpp_group_state(inst) == UNINITIALIZED`. + +`ccpp_final` itself is also silently idempotent for the same reason: the +first call's last-to-leave block deallocates `ccpp_suite_state`, so on a +single-instance host the unallocated state *is* the normal post-`ccpp_final` +condition. Both checks (`.not. allocated(ccpp_suite_state)` and +`ccpp_suite_state(inst) == CCPP_SUITE_UNREGISTERED`) silent-return rather +than erroring. By contrast, `ccpp_init`'s "not allocated" branch keeps +erroring with "ccpp_register has not been called" — there, the unallocated +state really does indicate a missed `ccpp_register` call. + +#### 7.2.1 State array allocation and instance indexing + +Each group cap declares an allocatable module-level array: + +```fortran +integer, private, allocatable :: ccpp_group_state(:) +``` + +Two generated subroutines manage it: + +```fortran +! Always takes number_of_instances as an explicit arg — never USEs a host module. +subroutine ccpp___state_alloc(number_of_instances, errmsg, errflg) + integer, intent(in) :: number_of_instances + ... + allocate(ccpp_group_state(number_of_instances)) + ccpp_group_state(:) = CCPP_GROUP_UNINITIALIZED + +subroutine ccpp___state_dealloc(errmsg, errflg) + ... + if (allocated(ccpp_group_state)) deallocate(ccpp_group_state) +``` + +`state_alloc` is called from the suite cap's `_init` subroutine. The count is +passed as an explicit argument: the local name of `number_of_instances` from host +metadata (multi-instance), or the integer literal `1` (single-instance): + +```fortran +! Multi-instance (host provides number_of_instances with local name ninstances): +subroutine test_suite_init(ninstances, errmsg, errflg) + integer, intent(in) :: ninstances + ... + call ccpp_test_suite_physics_state_alloc(ninstances, errmsg, errflg) + +! Single-instance (no number_of_instances in host metadata): +subroutine test_suite_init(errmsg, errflg) + ... + call ccpp_test_suite_physics_state_alloc(1, errmsg, errflg) +``` + +State array **indexing** in the phase subroutines uses the local name of +`instance_number` (e.g. `inst_num`) when the host provides it, otherwise the literal +`1`: + +```fortran +subroutine ccpp___init(inst_num, ...) + if (ccpp_group_state(inst_num) >= CCPP_GROUP_INITIALIZED) return + ... + ccpp_group_state(inst_num) = CCPP_GROUP_INITIALIZED +``` + +`instance_number` is injected into the `_init` and `_final` phase subroutine signatures +even when no scheme in those phases uses it directly — the state guard and state +transition require it. It does **not** appear in `_run`, `_timestep_init`, or +`_timestep_final` unless a scheme in those phases explicitly requests it. + +These two integer arrays replace both the boolean `initialized(:)` array from prebuild +and the string-based `ccpp_suite_state` from CAM-SIMA. + +--- + +## 8. Scheme Metadata and Variable Matching + +### 8.1 Scheme metadata structure + +Each scheme source file has a companion `.meta` file with `type = scheme` tables — one +table per public phase subroutine (`scheme_name_init`, `scheme_name_run`, +`scheme_name_timestep_init`, etc.). The section header for each variable entry is the +**local variable name** as it appears in the scheme's Fortran subroutine argument list. + +The internal metadata store is keyed as: + +``` +metadata[scheme_name][phase][standard_name] → {local_name, units, kind, dimensions, + intent, optional, active, ...} +``` + +Keying by `scheme_name` then `phase` enables cross-phase queries ("does this scheme +have a register phase?", "what are all variables of scheme X?") and matches the +conceptual model of the suite XML. + +### 8.2 Reading order + +All metadata files (host + scheme + DDT) are read in one pass without resolving DDT +type references. After the full read, the generator builds the known DDT list, then +resolves all type references. This avoids ordering dependencies between metadata files. + +### 8.3 Known DDT list + +After reading all metadata, the generator assembles the set of known DDT types from +`type = ddt` tables. Two categories: + +**Framework-defined DDTs**: declared in `type = ddt` metadata tables. The generator +knows their fields, dimensions, and access paths. + +**External DDTs**: types from external libraries (MPI, ESMF, etc.) that the generator +cannot introspect. These are declared in variable entries using an extended `type` +syntax: + +```ini +[mycomm] + standard_name = mpi_communicator + type = external:mpi_f08:mpi_comm + ... +``` + +The format is `external::`. The generator emits +`use mpi_f08, only: mpi_comm` and treats the variable as an opaque type — no field +traversal, no dimension indexing beyond what the metadata declares. + +### 8.4 Variable matching: scheme vs. host + +For each argument in a scheme's phase function (looked up by standard name): + +1. **Found in host+control flat dict** → use the resolved access path. If `units` or + `kind` differ from what the scheme declares, generate a transformation (Section 9). +2. **Not found, first use is `intent(out)`** → suite-owned variable. Add to suite data, + generate declaration in `ccpp__data.F90`. +3. **Not found, first use is `intent(in)` or `intent(inout)`** → **error**: variable + used before it is provided by any scheme or host. +4. **Found in suite data (from a prior scheme)** → use the suite data access path. + Apply transformation if needed. + +### 8.5 Cap call argument construction + +The generator builds the argument list for each scheme call from the scheme's metadata +argument order. For each argument: + +- **Direct pass-through** (no transformation, not optional): inline host access + expression — no local variable declared +- **Transformation**: cap-local temporary named after the scheme's local variable name + (from scheme metadata section header); see Section 10.2 for naming rules +- **Optional**: cap-local pointer array named `_p`; see Section 10.3 +- **Optional + transformation**: combined in the `if (active) then` block + +The generator does not parse Fortran source. All local names, types, kinds, dimensions, +and intents come exclusively from metadata. + +--- + +## 9. Variable Resolution and Access Path Construction + +### 9.1 Flat storage model (host+control+suite) + +The generator flattens the DDT hierarchy at parse time. After parsing, all host, +control, and suite variables are stored in a flat dictionary keyed by standard name. +Each entry contains: +- The Fortran local name +- The fully-qualified access path (e.g., `gfs_statein(instance_number)%phii`) +- The module to USE +- Dimension information with registered/arbitrary classification + +The DDT hierarchy is discarded after the flat dict is built. No live DDT object tree +is maintained during code generation. + +### 9.2 Access path construction + +For each variable, the generator constructs the call-site expression by applying +dimension rules to each dimension in order: + +1. **Registered scalar-index dim** (key in `SCALAR_INDEX_DIMS`; currently + `number_of_instances` → `instance_number`, + `number_of_threads` → `thread_number`) → scalar extraction using the + paired index variable's local Fortran name. Only permitted on + container DDT-instance variables, never on leaves (Rule 2; see + `capgen/metadata/registered_dimensions.py`). +2. **`horizontal_dimension`** → always substitute `horizontal_loop_begin:horizontal_loop_end` + (using control variable local names) at scheme call sites. For suite-owned array + allocation sizing, `horizontal_dimension` from the host `type=host` table is used directly. +3. **`vertical_*`** → substitute `1:local_vertical_dimension` +4. **Arbitrary dimension** → resolve to local name via its own metadata entry, emit + `1:local_name` +5. **`active` condition** → generate optional pointer-association guard (see Section 10.3) + +### 9.3 Module USE + +For each variable used in a group cap, the generator emits a `use module, only: varname` +statement. The module name comes from the enclosing `[ccpp-table-properties]` block name. +For suite-owned variables, the module is the generated `ccpp__data`. + +### 9.4 Eliminating TYPEDEFS_NEW_METADATA + +The manually-maintained `TYPEDEFS_NEW_METADATA` Python dict from prebuild is eliminated. +All information previously in that dict is now in metadata: +- The DDT type structure → `type = ddt` table +- The module-level instance → variable entry in the `type = host` table, module + implied by enclosing table name + +--- + +## 10. Variable Transformations and Optional Variables + +Variable transformations (unit/kind conversions) and optional variable handling are +combined — both occur within the same `if (active) then` block when a variable is +optional. + +### 10.1 Supported transformations + +- **Unit conversions** (e.g., Pa → hPa): formula from built-in conversion table (shared + with validator), keyed on source/target unit pair from metadata `units` attribute +- **Kind conversions** (e.g., r8 → kind_phys): from `kind` metadata attribute comparison + +The transformation framework is generic and pluggable — additional transformation types +(e.g., vertical flipping) can be added without restructuring the generator. + +### 10.2 Local variable naming + +The local variable name for a transformation temporary or optional pointer is derived +from the **scheme's local variable name** as declared in the scheme's metadata section +header (e.g., `[phii]` → local name is `phii`). The generator has this name without +parsing any Fortran. + +- Transformation temporary: scheme's local name + `_l` (e.g., `phii_l`) +- Optional pointer: scheme's local name + `_p` (e.g., `phii_p`) +- Conflict resolution: if two schemes in the same group cap use the same local name for + different standard names, append a numeric suffix before the suffix (e.g., `phii_2_l`) + +The generator validates that all generated local variable names and generated subroutine +names stay within Fortran's 63-character identifier limit. Violations are code-generation +errors — the developer must use a shorter local name in their metadata. + +The `active` expression in metadata is a Fortran logical expression written using +**CCPP standard names** (not local names). The generator translates all standard names +in the expression to their local Fortran names before emitting. + +Transformations **always** use a local temporary variable. The host variable is never +modified in-place — required for bit-for-bit reproducibility and to leave host data +uncorrupted if an exception occurs. Every conversion line carries an inline Fortran +comment (e.g., `! unit conversion: Pa to hPa`). Transformation mismatches (unknown +unit pair, unknown kind pair) are code-generation errors — no stdout/stderr from +generated code. + +### 10.3 The four cases + +The generator handles exactly four combinations per variable: + +**Case 1: No pointer, no transformation** (not optional, no unit/kind mismatch) + +No local variable is declared. The host access expression is used inline at the call +site: +```fortran +call scheme_run(..., gfs_statein(instance_number)%phii(lb:ub,1:nlevs), ...) +``` + +**Case 2: Pointer only** (optional, no transformation) + +Pointer array declared at function top; conditional association in `if (active)` block: +```fortran +! declaration: +type(real_kind_phys_rank1_ptr_type), target :: phii_p(number_of_threads) + +! before call: +if () then + phii_p(thread_number)%ptr => gfs_statein(instance_number)%phii(lb:ub,1:nlevs) +else + nullify(phii_p(thread_number)%ptr) +end if + +call scheme_run(..., phii_p(thread_number)%ptr, ...) + +! after call: +nullify(phii_p(thread_number)%ptr) +``` + +**Case 3: Transformation only** (not optional, unit/kind mismatch) + +Local temporary `phii_l` (scheme's local name + `_l`); intent-driven emission: + +| `intent` | Pre-call | Post-call | +|---|---|---| +| `in` | `phii_l = host_phii(...) * factor ! unit conversion: X to Y` | nothing | +| `out` | nothing | `host_phii(...) = phii_l / factor ! unit conversion: Y to X` | +| `inout` | pre-call as above | post-call as above | + +```fortran +! declaration: +real(kind=kind_phys) :: phii_l(lb:ub, 1:nlevs) ! or appropriate rank/kind + +! before call (intent in/inout): +phii_l = gfs_statein(instance_number)%phii(lb:ub,1:nlevs) * 0.01_kind_phys ! unit conversion: Pa to hPa + +call scheme_run(..., phii_l, ...) + +! after call (intent inout/out): +gfs_statein(instance_number)%phii(lb:ub,1:nlevs) = phii_l * 100.0_kind_phys ! unit conversion: hPa to Pa +``` + +**Case 4: Pointer and transformation** (optional + unit/kind mismatch) + +Two local variables: `phii_l` (transformation temporary) and `phii_p` (pointer array). +Sequence: (1) apply transformation to `phii_l` depending on intent, (2) assign pointer +to `phii_l`, (3) call scheme, (4) nullify pointer, (5) apply back-transformation from +`phii_l` to host depending on intent. All within the `if (active)` block: + +```fortran +! declarations: +real(kind=kind_phys) :: phii_l(lb:ub, 1:nlevs) +type(real_kind_phys_rank1_ptr_type), target :: phii_p(number_of_threads) + +! before call: +if () then + ! step 1: apply forward transformation (intent in/inout) + phii_l = gfs_statein(instance_number)%phii(lb:ub,1:nlevs) * 0.01_kind_phys ! unit conversion: Pa to hPa + ! step 2: assign pointer to transformed local + phii_p(thread_number)%ptr => phii_l +else + nullify(phii_p(thread_number)%ptr) +end if + +call scheme_run(..., phii_p(thread_number)%ptr, ...) + +! after call: +if () then + ! step 4: nullify pointer + nullify(phii_p(thread_number)%ptr) + ! step 5: apply back-transformation (intent inout/out) + gfs_statein(instance_number)%phii(lb:ub,1:nlevs) = phii_l * 100.0_kind_phys ! unit conversion: hPa to Pa +end if +``` + +The wrapper types (`real_kind_phys_rank1_ptr_type` etc.) are defined once in the +generated shared types module (`ccpp__types.F90`), not re-declared inside every +group cap function. Passing an unassociated pointer is safe under all compiler modes. + +--- + +## 11. Subcycle Loops + +When a group in the suite XML contains ``, the generator emits a +Fortran `do` loop in the group cap: + +```fortran +do = 1, N ! subcycle: N iterations from suite XML + ! ... scheme calls ... +end do +``` + +`ccpp_loop_counter` and `ccpp_loop_extent` are **loop-context variables** — a special +class that does not fit any of the five table types: +- NOT `type = control`: not passed as `ccpp_physics_*` arguments from the host +- NOT `type = host`: not from host module USE +- NOT `type = suite`: not persistent allocated data + +The generator has built-in knowledge of these two standard names. `ccpp_loop_extent` +value comes from the `loop=N` attribute in the suite XML definition file. +`ccpp_loop_counter` is the do loop induction variable. Both exist only within the +generated loop scope — they are not in scope outside a subcycle block. + +Any scheme that requests `ccpp_loop_counter` or `ccpp_loop_extent` by standard name +receives the loop variables at the call site. Their local names in the cap are derived +from the scheme's metadata section headers as for any other variable (Section 10.2). + +--- + +## 12. Init/Finalize Deduplication + +If the same scheme appears more than once within a single group (e.g., via subcycles), +having its `_init` called multiple times is a **code generator bug** — the generator +**errors out** rather than silently deduplicating. The suite XML must not list the same +scheme multiple times in the same group for non-run phases. + +If the same scheme appears in multiple groups, its `_init` is called once per group. +This is acceptable — idempotency is a contract all scheme `_init` routines must satisfy. + +The same rule applies to `_timestep_init`, `_timestep_final`, and `_final`. + +--- + +## 13. Suite-Owned Data + +### 13.1 Discovery + +The generator identifies suite-owned variables during variable resolution: variables +requested by schemes that are not satisfied by host metadata (`type = host` or +`type = control`). + +**Error condition**: if a variable is determined to be suite-owned (not provided by the +host) and the first scheme that uses it does not have it as `intent(out)`, the generator +errors out. A suite-owned variable that is first read before it is written would be +used uninitialized. + +### 13.2 Generated files and allocation + +Suite-owned variables are declared in a generated suite data module +(`ccpp__data.F90`) as fields of a Fortran DDT. The suite cap and all group caps +`use` this module. + +Allocation happens in `ccpp_init` (suite cap). Deallocation happens in `ccpp_final`. +Suite-owned arrays are allocated for the **full `horizontal_dimension`** — threads +access their respective horizontal chunk (`horizontal_loop_begin:horizontal_loop_end`) +at scheme call sites. This avoids per-call allocation overhead. If this proves to +consume too much memory at scale, a future revision may move allocation to per-phase +with chunk-sized arrays; start with the full-dimension approach. + +Subsetting (applying horizontal loop bounds, instance index) happens at scheme call +sites in the group cap, not in the suite cap. + +### 13.3 Metadata + +The generator also writes a `type = suite` metadata table (`ccpp__data.meta`) +as a byproduct. The `_data` suffix matches the companion Fortran file +`ccpp__data.F90`, satisfying the `.meta` ↔ `.F90` filename pairing. This +file is for inspection and debugging — it is not consumed by the generator on +subsequent runs. + +--- + +## 14. Constituent API + +> **Status (2026-05-12).** The constituent API in capgen has evolved past +> the sketch below. The current implementation is: +> +> - One `ccpp_model_constituents_obj(:)` array (sized to +> `number_of_instances`), declared and owned by the **generator** in +> `ccpp_host_constituents.F90` — not by the host. +> - Host-facing API: `ccpp_register_constituents(host_constituents, +> instance_number, ...)`, `ccpp_initialize_constituents`, +> `ccpp_number_constituents`, `ccpp_const_get_index`, +> `ccpp_constituents_array(instance_number)`, etc. All per-instance. +> - Schemes follow four rules: register-phase +> `ccpp_constituent_properties_t(:), intent=out, allocatable`; +> physics-phase consume via `advected=true intent=in/inout`; tendency +> produce via `constituent=true intent=out` + `tendency_of_` +> std_name; mismatched combos are codegen errors. +> - **Authoritative reference**: `doc/constituents.md` (full lifecycle + +> API + examples). +> - **Architecture review and proposed reforms**: `doc/constituents_overhaul.md` +> (2026-05-12, meeting-quality discussion of original capgen vs +> capgen vs cam-sima needs, bugs/flaws, class-A/B property +> classification, three proposals A/B/C). +> +> The historic text below is retained for context but does not describe +> the live system. + +### 14.1 Type definition (historic) + +`ccpp_model_constituents_t` is unchanged from CAM-SIMA. The type definition lives in +the framework library (`ccpp_constituent_prop_mod`), not in generated code. + +### 14.2 Ownership and lifecycle (historic — superseded) + +The **host model** declares and owns the constituent object: + +```fortran +use ccpp_constituent_prop_mod, only: ccpp_model_constituents_t +type(ccpp_model_constituents_t) :: constituents ! unallocated initially +``` + +The host passes it to `ccpp_register`, which allocates and populates it. After +`ccpp_register` returns, the host holds a fully allocated object ready for `lock_table`, +`const_index`, `copy_in`, `copy_out`. + +The `constituents` argument to `ccpp_register` is mandatory. + +### 14.3 Register phase mechanics (historic — superseded) + +The suite cap's register routine: +1. Iterates over all constituent-providing schemes in the suite +2. Calls each scheme's `_register` entrypoint, which returns a + `ccpp_constituent_properties_t` array +3. Collects these arrays and populates the constituent object + +No `group_name` dispatch is needed for register — it operates on the whole suite. + +--- + +## 15. Generated Output Files + +All files are written to `--output-root`. + +| File | Contents | +|---|---| +| `ccpp_kinds.F90` | Kind parameter definitions. **Always generated.** Re-exports specs from `iso_fortran_env` (default) or host-supplied modules as `integer, parameter, public :: = `. If no `--kind-type` is supplied, `kind_phys=iso_fortran_env:REAL64` is injected automatically (logged at INFO). | +| `_ccpp_cap.F90` | Static API — host imports, suite_name dispatch (filename and module name derived from `--host-name`) | +| `ccpp__cap.F90` | Suite cap — suite data import, state machine, group dispatch | +| `ccpp___cap.F90` | Group cap — scheme call sites, state array, optionals, transformations. USEs `ccpp_kinds` for any kind referenced in transformation temporaries. | +| `ccpp__data.F90` | Suite data module — framework-owned interstitial DDT. USEs `ccpp_kinds` for any kind referenced in suite-var declarations. | +| `ccpp__types.F90` | Shared cap types — optional pointer wrapper types, transformation locals. USEs `ccpp_kinds` for any kind referenced in pointer wrappers. | +| `ccpp__data.meta` | Generated `type = suite` metadata table — pairs with `ccpp__data.F90` (output-only, for inspection) | +| `datatable.xml` | Generator database for `ccpp_datafile.py` queries. `ccpp_kinds.F90` appears under ``; `_ccpp_cap.F90` appears under ``. | + +`ccpp_kinds.F90` is a dependency of all generated Fortran files that reference any kind parameter (group cap, suite types, suite data). The static API and suite cap have no kind references and do not USE it. + +--- + +## 16. CLI Invocation + +``` +ccpp_capgen.py + --host-name + --host-files + --scheme-files + --suites + --output-root + --kind-type NAME=[MODULE:]SPEC # repeatable, see § 16.1 + --verbose # once = info; twice = debug +``` + +The generator also supports programmatic Python invocation (import and call directly), +using the same internal code paths as the CLI. + +### 16.1 Kind specifications + +Each `--kind-type` maps a CCPP-visible kind name to a Fortran precision constant. Syntax: + +``` +--kind-type =[:] +``` + +* `` — kind name as published in `ccpp_kinds` and referenced in scheme metadata + (e.g. `kind_phys`). +* `` — name of a precision constant (kind parameter) defined in some Fortran + module. +* `` — Fortran module that defines ``. **Optional**: when omitted, + `` must be a standard `ISO_FORTRAN_ENV` constant (`REAL32`, `REAL64`, `INT32`, + ...) and the module defaults to `iso_fortran_env`. If `` is not a known ISO + constant, omitting `` is an error. + +Examples: + +* `--kind-type kind_phys=REAL64` → + `use iso_fortran_env, only: REAL64; integer, parameter, public :: kind_phys = REAL64` +* `--kind-type kind_phys=my_host_kinds:kind_r8` → + `use my_host_kinds, only: kind_r8; integer, parameter, public :: kind_phys = kind_r8` + +The flag may be specified multiple times. `ccpp_kinds.F90` is **always generated**. If +no `--kind-type` is supplied (or `kind_phys` is omitted from a non-empty list), the +generator injects `kind_phys=iso_fortran_env:REAL64` and logs an INFO message. + +### 16.2 datatable.xml and ccpp_datafile.py + +The generator emits `datatable.xml` encoding the full relationships between suites, +groups, schemes, and variables. A separate query utility (`ccpp_datafile.py`) provides +a rich query interface used by CMake and other build systems. The full query surface of +the existing `ccpp_datafile.py` is preserved — the simplified `--ccpp-files` query is +one of many. + +The XML structure is: + +```xml + + + + /abs/path/ccpp_kinds.F90 + + + /abs/path/_ccpp_cap.F90 + + + /abs/path/ccpp__cap.F90 + ... + + + + + + + + ... + + + + + ... + + + + + + + ... + + + + + + /abs/path/to/dep.F90 + ... + + +``` + +The `` section is populated from the `dependencies` table-level property +of all scheme metadata files (Section 3.5). Paths are resolved to absolute paths at +generation time, then sorted and deduplicated before writing. + +### 16.3 CMake integration pattern + +The generator runs at CMake configure time via `execute_process`. Generated sources are +discovered by querying `ccpp_datafile.py --ccpp-files`. Host and scheme Fortran sources +are found by replacing `.meta` with `.F90` (same base name convention). + +--- + +## 17. Design Decisions Not Carried Forward + +The following patterns from prebuild or capgen are explicitly **not** carried forward: + +| Pattern | Reason | +|---|---| +| `ccpp_t` / `cdata` struct | Replaced by explicit named control variable arguments | +| `TYPEDEFS_NEW_METADATA` Python dict | Replaced by DDT instance declarations in metadata | +| String-based `ccpp_suite_state` | Replaced by integer state arrays with named parameters | +| Boolean `initialized(:)` array | Replaced by integer state arrays | +| Flat-field arguments to group caps | DDT arguments are used instead (as in prebuild) | +| Scope-chain variable promotion | Suite-owned variables explicitly discovered and declared | +| Fortran-vs-metadata validation in generator | Moved to standalone validator tool | +| Re-declaration of pointer wrapper types per function | Declared once in shared types module | +| `ccpp_physics_suite_init/finalize` | Replaced by `ccpp_init`/`ccpp_final` | +| `type = module` (capgen) | Renamed to `type = host` | +| `finalize` phase name | Renamed to `final` | +| Array size checks in caps | Not generated by default; rely on compiler bounds checking | +| Auto-clone of `is_constituent` scheme args into framework `%instantiate` calls | Replaced by explicit registration (host_constituents arg + register-phase `ccpp_constituent_properties_t`); the auto-clone path is also available behind the opt-in `--legacy-auto-clone-constituents` shim for legacy hosts (single-instance only — see `doc/auto_clone_constituents.md`) | +| `ConstituentVarDict` synthetic scope between suite and host | Removed; constituents are a `source='constituent'` classification on `ResolvedArg` | + +--- + +## 18. Outstanding Work + +See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` +(deferred items) for the canonical list. Snapshot as of 2026-05-13: + +### Landed in the 2026-05-12 session + +- **`instance_number` / `number_of_instances` paired opt-in** — hosts + may omit both for a single-instance API. +- **Module-name override** — `module_name = ` on + `[ccpp-table-properties]` for scheme/host/ddt. +- **Vertical-flip transform** — `top_at_one` per-var attribute; + composes with unit/kind transforms. +- **Multiple `dependencies = …` lines** per `[ccpp-table-properties]`. +- **Sliced local names** with long subscript-token CCPP standard names + no longer trip the 63-char Fortran-id limit. +- **Unit normalization** — `m2` ≡ `m+2` (and friends). +- **Subcycle bound = CCPP std name** — including DDT-component access + paths (`phys_state%num_subcycles`). +- **Nested ``** — preserved end-to-end as nested `do` loops. +- **Active-expression + subcycle bounds** included in introspection + inputs. +- **TARGET on `ccpp_suite_data(:)`** module-level array. +- **Group-state alloc idempotency** (matches suite-state alloc). +- **Framework PR**: `ccpt_deallocate` ownership tracking via + `framework_owns_me` flag. Backward-compatible. Landed in + capgen's vendored framework copy; still needs upstream merge + to ccpp-framework + original ccpp-capgen. +- **Identity unit conversions** no longer emit misleading "unit + conversion: kind_phys to kind_phys" comment. +- **Improved duplicate-standard-name error** lists both colliding + access paths. +- **Suite-level `` / ``** SDF elements consumed: named + scheme's init/final phase emitted inside `_init` / + `_final`. Single scheme only; long-form spellings + (``, ``, ``) rejected. +- **Constituent resolver — host metadata wins**: hosts that declare + framework-named std_names (`ccpp_constituents`, `index_of_`, ...) + short-circuit capgen's auto-provisioning so legacy hosts (GFS, + SCM) keep using their own short local names (e.g. `ntcw`) without + blowing Fortran's 63-char identifier limit. + +### Landed 2026-05-13 (morning + afternoon) + +- **`--legacy-mode` shim** — transient parse-time rewrite of legacy + CCPP standard names (`horizontal_loop_extent` → + `horizontal_dimension`). Available on `ccpp_capgen.py` and + `ccpp_validator.py`; loud banner at startup. Isolated in + `metadata/legacy_compat.py` and tagged `# legacy-compat:` for clean + removal once scheme metadata has been migrated. +- **`_FRAMEWORK_CONST_DIM_INPUTS` cleanup** — the hand-curated + frozenset in `generator/host_cap.py` was removed; framework- + constituent dimension references now ride on a dedicated + `used_const_dim_std_names` field on `ResolvedArg`. +- **`active` expression case-folding** — mixed-case standard names + in `active = (...)` are now lowercased at parse time so they match + the canonical lowercase host_dict keys (Fortran is case-insensitive, + so embedded logical operators are unaffected). + +### Landed 2026-05-13 (late evening — SCM-driven session) + +- **`SCALAR_INDEX_DIMS` registered table** — `metadata/registered_dimensions.py` + carries the single source of truth for count-dim ↔ scalar-index pairings + (`number_of_instances → instance_number`, `number_of_threads → thread_number`). + Drops the old `instance_dimension` placeholder. Rule 2 (leaves never carry + registered dims) enforced at parse time with rich error messages. +- **Loop-context resolver wired** — scheme args declaring + `ccpp_loop_counter` / `ccpp_loop_extent` resolve inside `` to + the generated do-loop locals (or the loop's literal/host-resolved + bound for the extent). Outside-subcycle raises a clear parse-time + error pointing at the SDF contract. +- **Write-if-changed** — every generated cap file goes through + `metadata/parse_tools/io_helpers.py::write_if_changed`; unchanged + files keep their mtime so CMake/Make/Ninja don't trigger a rebuild + cascade on regenerate. Staging temp lives next to the target under + `--output-root`, never `/tmp`. Logger emits `"Wrote …"` vs + `"Unchanged: …"`. +- **`--scheme-files` query** — `datatable.xml` carries a `` + section listing the user-supplied scheme `.F90` source paths actually + referenced by some loaded suite (group phases + suite-level + `` / `` hooks). Companion `` filter applied + to scheme tables (host/control/ddt deps still flow unconditionally). +- **`ccpp__cap.F90` group dispatch + `ccpp_physics_*` suite + dispatch case-default** — unknown `group_name` / `suite_name` now + sets `errflg=1` and writes a clear message, no silent fall-through. +- **Missing-scheme parse-time detection** — `resolve_suite` walks every + scheme reference in the SDF (group phases + `` + ``) + and raises with the full list when any aren't in the scheme store. + Replaces silent empty-group-cap emission. +- **Validator continuation look-ahead** — `_join_continuation` now + detects continuation when the *current* line has no trailing `&` + but the next line has a column-6 `&` marker (fixed-form F77). + Fixes `sfc_sice.f::sfc_sice_run` and similar legacy CCPP-physics + signatures. +- **Metadata error enrichment** — `MetaVar.set_attr` wraps every + `check_X` helper failure with variable name + attribute name + raw + value + source location, so `'' is not a valid unit` becomes + actionable across a 60+ file load. +- **Character pointer-wrapper name encodes len** — `_ptr_type_name` + bakes the length into the wrapper name so two `character(len=N)` + args of different lengths don't collide on a single + `character_rank1_ptr_type` symbol. `len=:` → `_deferred`; `len=*` + rejected; unparseable lengths rejected. +- **DDT-instance non-registered-dim diagnostic** — a DDT-instance + variable with dims none of which are registered scalar-index AND + with flattenable fields raises at parse time with the concrete + would-be-broken access pattern. Empty DDTs (e.g. + `ccpp_constituent_prop_ptr_t`) flow through unchanged. +- **`build_ddt_module_map` honors per-DDT `module_name` override** — + CCPP-physics `radsw_param.meta` declares `cmpfsw_type` in a + scheme-less file with explicit `module_name`; previously skipped. + Precedence: DDT's own `module_name` wins > co-located non-DDT + table's resolved module > skipped. +- **`_resolve_single_bound` substitutes scalar-idx placeholders** — + dim bounds that resolve through a per-thread/per-instance DDT + field no longer leak the std-name placeholder in nested + subscripts. +- **Legacy-mode adds second pair** — `--legacy-mode` now also + rewrites `number_of_openmp_threads → number_of_threads`. Banner + enumerates every pair automatically; no hard-coded text per + pairing. + +### Landed in the 2026-05-14 → 2026-05-20 window + +- **`--no-host-introspection` flag** (2026-05-14) — stubs the bodies + of the five suite-introspection routines in `_ccpp_cap.F90`, + dropping the file from ~33k lines to ~800 for the 10-suite SCM + build (the case-blocks were making even `-O1` compilation + effectively hang). Signatures stay so existing host callers still + link; stubbed bodies return `errflg = 1` with a clear `errmsg`. +- **Per-instance dynamic-constituents buffer** (2026-05-18) — the + per-suite buffer that holds register-phase-allocated constituents + was lifted from "shared across instances" to a per-instance wrapper + DDT array. Surfaced by the new combined multi-instance + + constituents end-to-end test (`instances_advection`). Fixes a + latent set_const_index conflict and a class-B setter-mutation + problem. +- **Final-path silent idempotency** (2026-05-15) — both + `ccpp_physics_final` and `ccpp_final` return `errflg = 0` on + repeats; `ccpp_physics_final` additionally silent-skips after + `ccpp_final` teardown. +- **Validator per-arg attribute checks** (2026-05-20) — `intent`, + `type`, `kind`, and `rank` checked per argument; asymmetric + `optional` rule; DDT + `external::` normalisation; + character `len=*` wildcard. Caught 67 real metadata/Fortran + disagreements in the SCM physics tree on landing day. +- **Host/scheme metadata cross-checks** (late 2026-05-20) — resolver + enforces type identity, rank, and per-position dim entries between + host metadata (or first-writer suite-owned var) and every consuming + scheme arg. Bare `X` ≡ `1:X` ≡ `ccpp_constant_one:X` collapse; + every other lower bound stays distinct; numeric kind stays lenient + (transform path). +- **Host `active` + scheme arg runtime guard** (late 2026-05-20) — + replaces the earlier static rule that required scheme metadata to + declare `optional = True` for any host-active variable. Optional + scheme arg → pointer-association (PRESENT()-aware); non-optional + scheme arg → group-cap runtime guard before any transform. +- **`ccpp_static_api.F90` → `_ccpp_cap.F90`** — generated public + entry-point file is now per-host (filename and module name driven + by `--host-name`). Multiple host integrations can coexist in one + build. + +### Landed 2026-05-21 + +- **`--gfs-dim-aliases` shim** — transient CLI flag that treats GFS + radiation/composition vertical-dim names + (`adjusted_vertical_layer_dimension_for_radiation` and + `vertical_composition_dimension`) as equivalent to + `vertical_layer_dimension` **inside the resolver's + per-position dim-identity check only** (upper bound only). Host + variables stay distinct everywhere else. Single touchpoint at + `generator/suite_resolver.py::_canonical_dim`; module + `metadata/dim_aliases.py`; touchpoints tagged `# dim-aliases:` for + clean removal. Generator-only (the validator never reaches the + canonicaliser). Required for CCPP-SCM 17p8 to build under + capgen. +- **`--legacy-auto-clone-constituents` shim** — transient CLI flag + that reinstates original ccpp-capgen's auto-clone-static-constituent + registration path. Every `is_constituent` consumer scheme arg + (`advected = True` / `constituent = True` / `molar_mass = …`) with + no register-phase source is auto-registered into the per-suite + dynamic-constituents buffer using values lifted straight from the + scheme metadata. Adds four legacy `%instantiate` kwargs to the + parser (`default_value`, `min_value`, `water_species`, + `mixing_ratio_type`). Synthesises `long_name` from std_name when + missing; falls back `diag_name` to local_name; lifts `vertical_dim` + from the arg's dim list. Available on both `ccpp_capgen.py` and + `ccpp_validator.py`. **Single-instance only** — aborts before + parsing if the host declares `instance_number` + + `number_of_instances`. Module + `metadata/auto_clone_constituents.py`; touchpoints tagged + `# auto-clone-constituents:`. New e2e fixture + `end-to-end-tests/advection_auto_clone/` is a port of CAM-SIMA's + `advection_test`. Full reference: `doc/auto_clone_constituents.md`. + +### Test status + +- **Unit tests**: 1426 passing (1438 with doctests; as of 2026-06-01). + Run via `python unit-tests/run_tests.py [--doctest]`. +- **End-to-end tests** (10 passing): `advection`, + `advection_auto_clone`, `capgen`, `chunked_data`, `ddthost`, + `instances`, `instances_advection`, `nested_suite`, `opt_arg`, + `var_compat`. SCM running against ccpp-physics continues to be + the active driver — most of the landings since 2026-05-13 were + surfaced by SCM build/runtime failures. Tree is off-limits for + in-session edits — user-driven. + +### Still deferred + +- **Constituents overhaul** — discussion doc at + `doc/constituents_overhaul.md` (2026-05-12). Three proposals on the + table (A bugfix-only / B class-A/B split + setters / C host-only + registration). Pending decision in upcoming meeting. +- **Framework setter additions** — `set_advected`, `set_diagnostic_name`, + `set_default_value`, possibly `set_mixing_ratio_type`. Coordinated with + the overhaul. +- ~~**Validator host-metadata check**~~ — **Landed 2026-06-01**: + `ccpp_validator.py --host-files` validates `type=host` and `type=ddt` + tables against module-level decls and derived-type definitions in + the same `--source-files` Fortran tree. `type=control` is silent- + skipped; `type=scheme` in `--host-files` is a hard error. Per-arg + type/kind/rank checks reuse `_check_arg_attributes`. See + `doc/migration.md` §7.4. +- **Codegen-time scheme-registration cross-check** — new metadata attr + `registers_std_names = a, b, c` on register-phase tables; replaces + current runtime `int_unassigned` check with codegen-time error. +- **Suppress `ccpp_host_constituents.F90` when unused** — currently + emitted for every build; now *correct* (empty) for SCM-style hosts + thanks to the host-wins rule, but still dead code. +- **`--legacy-mode` shim removal** — transient; remove + `metadata/legacy_compat.py`, `unit-tests/test_legacy_compat.py`, and + every `# legacy-compat:` touchpoint when scheme metadata has + migrated. +- **`--gfs-dim-aliases` shim removal** (added 2026-05-21) — + transient; remove `metadata/dim_aliases.py`, + `unit-tests/test_dim_aliases.py`, and every `# dim-aliases:` + touchpoint when GFS metadata stops spelling + `vertical_layer_dimension` as + `adjusted_vertical_layer_dimension_for_radiation` / + `vertical_composition_dimension`. +- **`--legacy-auto-clone-constituents` shim removal** (added + 2026-05-21) — transient; remove + `metadata/auto_clone_constituents.py`, + `unit-tests/test_auto_clone_constituents.py`, the sample fixtures + (`unit-tests/sample_files/scheme_auto_clone_consumer.meta`, + `unit-tests/sample_suite_files/suite_auto_clone.xml`), and every + `# auto-clone-constituents:` touchpoint when consumers have moved + to explicit `host_constituents(:)` declaration or register-phase + scheme registration. +- **Nested subcycle `ccpp_loop_counter` semantics**: a scheme inside a + nested subcycle requesting `ccpp_loop_counter` would get the + OUTERMOST counter, not the innermost. None of the cam-sima schemes + use this — revisit if a real scheme needs the innermost. +- **Python linter / formatter pass** — pick `ruff` and apply across + `capgen/`. +- **Generated Fortran ↔ Codee formatter idempotency** — emitted `.F90` + must round-trip cleanly through the project's Codee formatter. +- **`fortran_to_metadata` developer utility** — bootstrap a `.meta` + skeleton from an existing `.F90` subroutine. + +### Where to find the migration summary + +`doc/migration.md` — user-facing single-page summary of metadata + SDF ++ host-Fortran requirements after all the above changes. Read it +first when porting a host model. diff --git a/end-to-end-tests.sh b/end-to-end-tests.sh new file mode 100755 index 00000000..c9bc074e --- /dev/null +++ b/end-to-end-tests.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +mkdir -p build +rm -fr build/* +cd build +cmake ../end-to-end-tests +make +ctest +cd .. diff --git a/end-to-end-tests/CMakeLists.txt b/end-to-end-tests/CMakeLists.txt new file mode 100644 index 00000000..3ea171a8 --- /dev/null +++ b/end-to-end-tests/CMakeLists.txt @@ -0,0 +1,88 @@ +cmake_minimum_required(VERSION 3.18) + +project(ccpp_framework_end_to_end_tests + VERSION 8.0.0 + LANGUAGES Fortran) + +enable_testing() +include(cmake/ccpp_capgen.cmake) + +option(OPENMP "Enable OpenMP support for the framework" ON) +message(STATUS "OpenMP ${OPENMP}") + +set(CCPP_VERBOSITY "2" CACHE STRING "Verbosity level of output (default: 0)") + +# Set appropriate flags to help with debugging test issues +if(${CMAKE_Fortran_COMPILER_ID} STREQUAL "GNU") + ADD_COMPILE_OPTIONS(-fcheck=all) + ADD_COMPILE_OPTIONS(-fbacktrace) + ADD_COMPILE_OPTIONS(-ffpe-trap=zero) + ADD_COMPILE_OPTIONS(-finit-real=nan) + ADD_COMPILE_OPTIONS(-ggdb) + ADD_COMPILE_OPTIONS(-ffree-line-length-none) + ADD_COMPILE_OPTIONS(-cpp) +elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL "Intel") + ADD_COMPILE_OPTIONS(-fpe0) + ADD_COMPILE_OPTIONS(-warn) + ADD_COMPILE_OPTIONS(-traceback) + ADD_COMPILE_OPTIONS(-debug extended) + ADD_COMPILE_OPTIONS(-fpp) + ADD_COMPILE_OPTIONS(-diag-disable=10448) +elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL "IntelLLVM") + ADD_COMPILE_OPTIONS(-fpe0) + ADD_COMPILE_OPTIONS(-warn) + ADD_COMPILE_OPTIONS(-traceback) + ADD_COMPILE_OPTIONS(-debug full) + ADD_COMPILE_OPTIONS(-fpp) +elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "NVIDIA" OR ${CMAKE_Fortran_COMPILER_ID} STREQUAL "NVHPC") + ADD_COMPILE_OPTIONS(-Mnoipa) + ADD_COMPILE_OPTIONS(-traceback) + ADD_COMPILE_OPTIONS(-Mfree) + ADD_COMPILE_OPTIONS(-Ktrap=fp) + ADD_COMPILE_OPTIONS(-Mpreprocess) +else() + message (WARNING "This program may not be able to be compiled with compiler :${CMAKE_Fortran_COMPILER_ID}") +endif() + +#------------------------------------------------------------------------------ +# Set MPI flags for Fortran with MPI F08 interface +find_package(MPI COMPONENTS Fortran REQUIRED) +if(NOT MPI_Fortran_HAVE_F08_MODULE) + message(FATAL_ERROR "MPI implementation does not support the Fortran 2008 mpi_f08 interface") +endif() + +#------------------------------------------------------------------------------ +# Set OpenMP flags for C/C++/Fortran +if(OPENMP) + find_package(OpenMP REQUIRED) +endif() + +#------------------------------------------------------------------------------ +# Set a default build type if none was specified +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + message(STATUS "Setting build type to 'Debug' as none was specified.") + set(CMAKE_BUILD_TYPE Debug CACHE STRING "Choose the type of build." FORCE) +endif() + +#------------------------------------------------------------------------------ +# Run simple tests first +add_subdirectory(chunked_data) +add_subdirectory(opt_arg) + +#------------------------------------------------------------------------------ +# Run intermediate tests next +add_subdirectory(nested_suite) +add_subdirectory(ddthost) +add_subdirectory(instances) + +#------------------------------------------------------------------------------ +# Run most complex tests last +add_subdirectory(capgen) +add_subdirectory(var_compat) +add_subdirectory(suite_allocate) +add_subdirectory(constituents_dim) +add_subdirectory(advection) +add_subdirectory(instances_advection) +# auto-clone-constituents: Remove this test when the legacy switch to support +# auto-clone constituents is removed (i.e. after CAM-SIMA updated its physics) +add_subdirectory(advection_auto_clone) diff --git a/test_prebuild/test_chunked_data/README.md b/end-to-end-tests/README.md.tobeupdated similarity index 97% rename from test_prebuild/test_chunked_data/README.md rename to end-to-end-tests/README.md.tobeupdated index 16db6fc5..a9f05bd7 100644 --- a/test_prebuild/test_chunked_data/README.md +++ b/end-to-end-tests/README.md.tobeupdated @@ -4,7 +4,7 @@ 2. Run the following commands: ``` cd test_prebuild/test_chunked_data/ -rm -fr build +#rm -fr build mkdir build ../../scripts/ccpp_prebuild.py --config=ccpp_prebuild_config.py --builddir=build cd build diff --git a/end-to-end-tests/advection/CMakeLists.txt b/end-to-end-tests/advection/CMakeLists.txt new file mode 100644 index 00000000..2b69dda0 --- /dev/null +++ b/end-to-end-tests/advection/CMakeLists.txt @@ -0,0 +1,80 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "cld_liq" "cld_ice" "apply_constituent_tendencies" "const_indices") +set(HOST_FILES "test_host_data" "test_host_mod" "test_host") +set(SUITE_FILES "cld_suite.xml") +set(HOST "test_host") +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") + +# Enable trace output in auto-generated caps +set(CCPP_TRACE ON) + +# Run ccpp_capgen +ccpp_capgen(TRACE ${CCPP_TRACE} + VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +# Add extra files needed for testing +set(EXTRA_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/../utils/test_utils.F90 +) + +add_executable(test_advection.x + ${EXTRA_FILES} + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES_FILTERED} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES} + test_advection_host_integration.F90 +) +target_link_libraries(test_advection.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_advection.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_advection.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_advection + COMMAND test_advection.x) diff --git a/test/advection_test/README.md b/end-to-end-tests/advection/README.md similarity index 54% rename from test/advection_test/README.md rename to end-to-end-tests/advection/README.md index 6dd53e9f..c460e13a 100644 --- a/test/advection_test/README.md +++ b/end-to-end-tests/advection/README.md @@ -1,21 +1,10 @@ # Advection Test Contains tests to exercise the capabilities of the constituents object, including: -- Adding a build-time constituent via metadata property -- Adding a run-time constituent via a register phase +- Adding run-time constituents from the host via a register phase +- Adding run-time constituents from schemes via a register phase - Also tests that trying to add a constituent outside of the register phase errors as expected - Passing around and modifying the constituent array - Accessing and modifying a constituent tendency variable - Passing around the constituent tendency array - Dimensions are case-insensitive - -## Building/Running - -To explicitly build/run the advection test host, run: - -```bash -$ cmake --fresh -S -B -DCCPP_RUN_ADVECTION_TEST=ON -$ cd -$ make -$ ctest -``` diff --git a/test/advection_test/advection_test_reports.py b/end-to-end-tests/advection/advection_test_reports.py similarity index 100% rename from test/advection_test/advection_test_reports.py rename to end-to-end-tests/advection/advection_test_reports.py diff --git a/test/advection_test/apply_constituent_tendencies.F90 b/end-to-end-tests/advection/apply_constituent_tendencies.F90 similarity index 100% rename from test/advection_test/apply_constituent_tendencies.F90 rename to end-to-end-tests/advection/apply_constituent_tendencies.F90 diff --git a/test/advection_test/apply_constituent_tendencies.meta b/end-to-end-tests/advection/apply_constituent_tendencies.meta similarity index 83% rename from test/advection_test/apply_constituent_tendencies.meta rename to end-to-end-tests/advection/apply_constituent_tendencies.meta index b7645a1b..ac02e5e4 100644 --- a/test/advection_test/apply_constituent_tendencies.meta +++ b/end-to-end-tests/advection/apply_constituent_tendencies.meta @@ -10,14 +10,14 @@ long_name = ccpp constituent tendencies units = none type = real | kind = kind_phys - dimensions = (horizontal_loop_extent, vertical_layer_dimension, number_of_ccpp_constituents) + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) intent = inout [ const ] standard_name = ccpp_constituents long_name = ccpp constituents units = none type = real | kind = kind_phys - dimensions = (horizontal_loop_extent, vertical_layer_dimension, number_of_ccpp_constituents) + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) intent = inout [ errcode ] standard_name = ccpp_error_code diff --git a/test/advection_test/cld_ice.F90 b/end-to-end-tests/advection/cld_ice.F90 similarity index 90% rename from test/advection_test/cld_ice.F90 rename to end-to-end-tests/advection/cld_ice.F90 index 3ace2f91..e3fc2abd 100644 --- a/test/advection_test/cld_ice.F90 +++ b/end-to-end-tests/advection/cld_ice.F90 @@ -49,7 +49,7 @@ end subroutine cld_ice_register !! \htmlinclude arg_table_cld_ice_run.html !! subroutine cld_ice_run(ncol, timestep, temp, qv, ps, cld_ice_array, & - errmsg, errflg) + errmsg, errcode) integer, intent(in) :: ncol real(kind=kind_phys), intent(in) :: timestep @@ -58,7 +58,7 @@ subroutine cld_ice_run(ncol, timestep, temp, qv, ps, cld_ice_array, & real(kind=kind_phys), intent(in) :: ps(:) real(kind=kind_phys), intent(inout) :: cld_ice_array(:, :) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode !---------------------------------------------------------------- integer :: icol @@ -66,7 +66,7 @@ subroutine cld_ice_run(ncol, timestep, temp, qv, ps, cld_ice_array, & real(kind=kind_phys) :: frz errmsg = '' - errflg = 0 + errcode = 0 ! Apply state-of-the-art thermodynamics :) do icol = 1, ncol @@ -87,16 +87,14 @@ end subroutine cld_ice_run !> \section arg_table_cld_ice_init Argument Table !! \htmlinclude arg_table_cld_ice_init.html !! - subroutine cld_ice_init(tfreeze, cld_ice_array, errmsg, errflg) + subroutine cld_ice_init(tfreeze, errmsg, errcode) real(kind=kind_phys), intent(in) :: tfreeze - real(kind=kind_phys), intent(inout) :: cld_ice_array(:, :) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode errmsg = '' - errflg = 0 - cld_ice_array = 0.0_kind_phys + errcode = 0 tcld = tfreeze - 20.0_kind_phys end subroutine cld_ice_init @@ -111,13 +109,13 @@ end subroutine cld_ice_init !! and the subroutine are parsed correctly. !! @{ - subroutine cld_ice_final(errmsg, errflg) + subroutine cld_ice_final(errmsg, errcode) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode errmsg = '' - errflg = 0 + errcode = 0 end subroutine cld_ice_final diff --git a/test/unit_tests/sample_scheme_files/temp_adjust.meta b/end-to-end-tests/advection/cld_ice.meta similarity index 70% rename from test/unit_tests/sample_scheme_files/temp_adjust.meta rename to end-to-end-tests/advection/cld_ice.meta index 4b96e316..2200f2c8 100644 --- a/test/unit_tests/sample_scheme_files/temp_adjust.meta +++ b/end-to-end-tests/advection/cld_ice.meta @@ -1,24 +1,18 @@ +# cld_ice is a scheme that produces a cloud ice amount [ccpp-table-properties] - name = temp_adjust + name = cld_ice type = scheme - -######################################################################## + [ccpp-arg-table] - name = temp_adjust_register + name = cld_ice_register type = scheme -[ config_var ] - standard_name = configuration_variable - type = logical +[ dyn_const_ice ] + standard_name = dynamic_constituents_for_cld_ice units = none - dimensions = () - intent = in -[ dyn_const ] - standard_name = dynamic_constituents_for_temp_adjust + dimensions = (:) + allocatable = True type = ccpp_constituent_properties_t - units = none - dimensions = () intent = out - allocatable = True [ errmsg ] standard_name = ccpp_error_message long_name = Error message for error handling in CCPP @@ -27,19 +21,19 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 dimensions = () type = integer intent = out -######################################################################## + [ccpp-arg-table] - name = temp_adjust_run + name = cld_ice_run type = scheme -[ foo ] - standard_name = horizontal_loop_extent +[ ncol ] + standard_name = horizontal_dimension type = integer units = count dimensions = () @@ -52,34 +46,33 @@ type = real kind = kind_phys intent = in -[ temp_prev ] - standard_name = potential_temperature_at_previous_timestep +[ temp ] + standard_name = temperature units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[ temp_layer ] - standard_name = potential_temperature - units = K - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = inout [ qv ] standard_name = water_vapor_specific_humidity units = kg kg-1 - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = inout [ ps ] standard_name = surface_air_pressure - state_variable = true type = real kind = kind_phys units = Pa - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) + intent = in +[ cld_ice_array ] + standard_name = cloud_ice_dry_mixing_ratio + advected = .true. + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys intent = inout [ errmsg ] standard_name = ccpp_error_message @@ -89,16 +82,24 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 dimensions = () type = integer intent = out + [ccpp-arg-table] - name = temp_adjust_init + name = cld_ice_init type = scheme +[ tfreeze ] + standard_name = water_temperature_at_freezing + long_name = Freezing temperature of water at sea level + units = K + dimensions = () + type = real | kind = kind_phys + intent = in [ errmsg ] standard_name = ccpp_error_message long_name = Error message for error handling in CCPP @@ -107,15 +108,16 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 dimensions = () type = integer intent = out + [ccpp-arg-table] - name = temp_adjust_finalize + name = cld_ice_final type = scheme [ errmsg ] standard_name = ccpp_error_message @@ -125,7 +127,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 diff --git a/end-to-end-tests/advection/cld_liq.F90 b/end-to-end-tests/advection/cld_liq.F90 new file mode 100644 index 00000000..c0d00a43 --- /dev/null +++ b/end-to-end-tests/advection/cld_liq.F90 @@ -0,0 +1,107 @@ +! Test parameterization with advected species +! + +module cld_liq + + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + implicit none + private + + public :: cld_liq_register + public :: cld_liq_init + public :: cld_liq_run + +contains + + !> \section arg_table_cld_liq_register Argument Table + !! \htmlinclude arg_table_cld_liq_register.html + !! + subroutine cld_liq_register(dyn_const, errmsg, errcode) + type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errcode + + errmsg = '' + errcode = 0 + allocate(dyn_const(2), stat=errcode) + if (errcode /= 0) then + errmsg = 'Error allocating dyn_const in cld_liq_register' + return + end if + call dyn_const(1)%instantiate(std_name="dyn_const3_wrt_moist_air_and_condensed_water", long_name='dyn const3', & + diag_name='DYNCONST3', units='kg kg-1', default_value=1._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + water_species=.true., mixing_ratio_type='dry', & + errcode=errcode, errmsg=errmsg) + call dyn_const(2)%instantiate(std_name="cloud_liquid_dry_mixing_ratio", long_name='Cloud liquid dry mixing ratio', & + diag_name='CLDLIQ', units='kg kg-1', default_value=0._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + ! Defer setting water_species later in the test + !water_species=.true., + mixing_ratio_type='dry', & + errcode=errcode, errmsg=errmsg) + + end subroutine cld_liq_register + + !> \section arg_table_cld_liq_run Argument Table + !! \htmlinclude arg_table_cld_liq_run.html + !! + subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, & + cld_liq_array, cld_liq_tend, errmsg, errcode) + + integer, intent(in) :: ncol + real(kind=kind_phys), intent(in) :: timestep + real(kind=kind_phys), intent(in) :: tcld + real(kind=kind_phys), intent(inout) :: temp(:, :) + real(kind=kind_phys), intent(inout) :: qv(:, :) + real(kind=kind_phys), intent(in) :: ps(:) + real(kind=kind_phys), intent(inout) :: cld_liq_array(:, :) + real(kind=kind_phys), intent(out) :: cld_liq_tend(:, :) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errcode + !---------------------------------------------------------------- + + integer :: icol + integer :: ilev + real(kind=kind_phys) :: cond + + errmsg = '' + errcode = 0 + + ! Apply state-of-the-art thermodynamics :) + do icol = 1, ncol + do ilev = 1, size(temp, 2) + cld_liq_array(icol, ilev) = max(0.0_kind_phys, cld_liq_array(icol, ilev)) + if ((qv(icol, ilev) > 0.0_kind_phys) .and. & + (temp(icol, ilev) <= tcld)) then + cond = min(qv(icol, ilev), 0.1_kind_phys) + cld_liq_tend(icol, ilev) = cond + qv(icol, ilev) = qv(icol, ilev) - cond + if (cond > 0.0_kind_phys) then + temp(icol, ilev) = temp(icol, ilev) + (cond * 5.0_kind_phys) + end if + end if + end do + end do + + end subroutine cld_liq_run + + !> \section arg_table_cld_liq_init Argument Table + !! \htmlinclude arg_table_cld_liq_init.html + !! + subroutine cld_liq_init(tfreeze, tcld, errmsg, errcode) + + real(kind=kind_phys), intent(in) :: tfreeze + real(kind=kind_phys), intent(out) :: tcld + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errcode + + errmsg = '' + errcode = 0 + tcld = tfreeze - 20.0_kind_phys + + end subroutine cld_liq_init + +end module cld_liq diff --git a/end-to-end-tests/advection/cld_liq.meta b/end-to-end-tests/advection/cld_liq.meta new file mode 100644 index 00000000..1abc40d0 --- /dev/null +++ b/end-to-end-tests/advection/cld_liq.meta @@ -0,0 +1,135 @@ +# cld_liq is a scheme that produces a cloud liquid amount +[ccpp-table-properties] + name = cld_liq + type = scheme +[ccpp-arg-table] + name = cld_liq_register + type = scheme +[ dyn_const ] + standard_name = dynamic_constituents_for_cld_liq + dimensions = (:) + type = ccpp_constituent_properties_t + intent = out + allocatable = true +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errcode ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = cld_liq_run + type = scheme +[ ncol ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ timestep ] + standard_name = time_step_for_physics + long_name = time step + units = s + dimensions = () + type = real + kind = kind_phys + intent = in +[ tcld] + standard_name = minimum_temperature_for_cloud_liquid + units = K + dimensions = () + type = real | kind = kind_phys + intent = in +[ temp ] + standard_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_LAYER_dimension) + type = real + kind = kind_phys + intent = inout +[ qv ] + standard_name = water_vapor_specific_humidity + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout +[ ps ] + standard_name = surface_air_pressure + type = real + kind = kind_phys + units = hPa + dimensions = (horizontal_dimension) + intent = in +[ cld_liq_array ] + standard_name = cloud_liquid_dry_mixing_ratio + diagnostic_name = CLDLIQ + advected = .true. + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = inout +[ cld_liq_tend ] + standard_name = tendency_of_cloud_liquid_dry_mixing_ratio + units = kg kg-1 s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = out + constituent = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errcode ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = cld_liq_init + type = scheme +[ tfreeze] + standard_name = water_temperature_at_freezing + long_name = Freezing temperature of water at sea level + units = K + dimensions = () + type = real | kind = kind_phys + intent = in +[ tcld] + standard_name = minimum_temperature_for_cloud_liquid + units = K + dimensions = () + type = real | kind = kind_phys + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errcode ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/test/advection_test/cld_suite.xml b/end-to-end-tests/advection/cld_suite.xml similarity index 100% rename from test/advection_test/cld_suite.xml rename to end-to-end-tests/advection/cld_suite.xml diff --git a/test/advection_test/cld_suite_error.xml b/end-to-end-tests/advection/cld_suite_error.xml similarity index 100% rename from test/advection_test/cld_suite_error.xml rename to end-to-end-tests/advection/cld_suite_error.xml diff --git a/test/advection_test/const_indices.F90 b/end-to-end-tests/advection/const_indices.F90 similarity index 84% rename from test/advection_test/const_indices.F90 rename to end-to-end-tests/advection/const_indices.F90 index bc3b46a7..5c77e29c 100644 --- a/test/advection_test/const_indices.F90 +++ b/end-to-end-tests/advection/const_indices.F90 @@ -17,7 +17,7 @@ module const_indices !! \htmlinclude arg_table_const_indices_run.html !! subroutine const_indices_run(const_std_name, num_consts, test_stdname_array, & - const_index, const_inds, errmsg, errflg) + const_index, const_inds, errmsg, errcode) use ccpp_constituent_prop_mod, only: int_unassigned use ccpp_scheme_utils, only: ccpp_constituent_index use ccpp_scheme_utils, only: ccpp_constituent_indices @@ -28,28 +28,28 @@ subroutine const_indices_run(const_std_name, num_consts, test_stdname_array, & integer, intent(out) :: const_index integer, intent(out) :: const_inds(:) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode !---------------------------------------------------------------- integer :: indx integer :: test_indx errmsg = '' - errflg = 0 + errcode = 0 ! Find the constituent index for - call ccpp_constituent_index(const_std_name, const_index, errflg, errmsg) - if (errflg == 0) then - call ccpp_constituent_indices(test_stdname_array, const_inds, errflg, errmsg) + call ccpp_constituent_index(const_std_name, const_index, errcode, errmsg) + if (errcode == 0) then + call ccpp_constituent_indices(test_stdname_array, const_inds, errcode, errmsg) end if ! Check that a non-registered constituent is detectable but ! does not cause an error - if (errflg == 0) then - call ccpp_constituent_index('unobtainium', test_indx, errflg, errmsg) + if (errcode == 0) then + call ccpp_constituent_index('unobtainium', test_indx, errcode, errmsg) if (test_indx /= int_unassigned) then - if (errflg == 0) then + if (errcode == 0) then ! Do not add an error if one is already reported - errflg = 2 + errcode = 2 write(errmsg, '(2a,i0,a,i0)') "ccpp_constituent_index called for ", & "'unobtainium' returned an index of ", test_indx, ", not ", & int_unassigned @@ -63,7 +63,7 @@ end subroutine const_indices_run !! \htmlinclude arg_table_const_indices_init.html !! subroutine const_indices_init(const_std_name, num_consts, test_stdname_array, & - const_index, const_inds, errmsg, errflg) + const_index, const_inds, errmsg, errcode) use ccpp_scheme_utils, only: ccpp_constituent_index, & ccpp_constituent_indices @@ -73,18 +73,18 @@ subroutine const_indices_init(const_std_name, num_consts, test_stdname_array, & integer, intent(out) :: const_index integer, intent(out) :: const_inds(:) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode !---------------------------------------------------------------- integer :: indx errmsg = '' - errflg = 0 + errcode = 0 ! Find the constituent index for - call ccpp_constituent_index(const_std_name, const_index, errflg, errmsg) - if (errflg == 0) then - call ccpp_constituent_indices(test_stdname_array, const_inds, errflg, errmsg) + call ccpp_constituent_index(const_std_name, const_index, errcode, errmsg) + if (errcode == 0) then + call ccpp_constituent_indices(test_stdname_array, const_inds, errcode, errmsg) end if end subroutine const_indices_init diff --git a/test/advection_test/const_indices.meta b/end-to-end-tests/advection/const_indices.meta similarity index 99% rename from test/advection_test/const_indices.meta rename to end-to-end-tests/advection/const_indices.meta index a4cc98e2..147e2ccb 100644 --- a/test/advection_test/const_indices.meta +++ b/end-to-end-tests/advection/const_indices.meta @@ -47,7 +47,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 @@ -99,7 +99,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 diff --git a/test/advection_test/dlc_liq.F90 b/end-to-end-tests/advection/dlc_liq.F90 similarity index 76% rename from test/advection_test/dlc_liq.F90 rename to end-to-end-tests/advection/dlc_liq.F90 index 20ff4b7b..134e3aed 100644 --- a/test/advection_test/dlc_liq.F90 +++ b/end-to-end-tests/advection/dlc_liq.F90 @@ -16,25 +16,25 @@ module dlc_liq !> \section arg_table_dlc_liq_init Argument Table !! \htmlinclude arg_table_dlc_liq_init.html !! - subroutine dlc_liq_init(dyn_const, errmsg, errflg) + subroutine dlc_liq_init(dyn_const, errmsg, errcode) type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode character(len=256) :: stdname errmsg = '' - errflg = 0 - allocate(dyn_const(1), stat=errflg) - if (errflg /= 0) then + errcode = 0 + allocate(dyn_const(1), stat=errcode) + if (errcode /= 0) then errmsg = 'Error allocating dyn_const in dlc_liq_init' return end if call dyn_const(1)%instantiate(std_name="dyn_const3", long_name='dyn const3', & diag_name='DYNCONST3', units='kg kg-1', default_value=1._kind_phys, & vertical_dim='vertical_layer_dimension', advected=.true., & - errcode=errflg, errmsg=errmsg) - call dyn_const(1)%standard_name(stdname, errcode=errflg, errmsg=errmsg) + errcode=errcode, errmsg=errmsg) + call dyn_const(1)%standard_name(stdname, errcode=errcode, errmsg=errmsg) end subroutine dlc_liq_init diff --git a/test/advection_test/dlc_liq.meta b/end-to-end-tests/advection/dlc_liq.meta similarity index 98% rename from test/advection_test/dlc_liq.meta rename to end-to-end-tests/advection/dlc_liq.meta index fedb6243..41a69db9 100644 --- a/test/advection_test/dlc_liq.meta +++ b/end-to-end-tests/advection/dlc_liq.meta @@ -20,7 +20,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 diff --git a/test/advection_test/test_advection_host_integration.F90 b/end-to-end-tests/advection/test_advection_host_integration.F90 similarity index 96% rename from test/advection_test/test_advection_host_integration.F90 rename to end-to-end-tests/advection/test_advection_host_integration.F90 index 0ee54da7..f1f73576 100644 --- a/test/advection_test/test_advection_host_integration.F90 +++ b/end-to-end-tests/advection/test_advection_host_integration.F90 @@ -7,7 +7,7 @@ program test implicit none character(len=cs), target :: test_parts1(1) - character(len=cm), target :: test_invars1(12) + character(len=cm), target :: test_invars1(11) character(len=cm), target :: test_outvars1(13) character(len=cm), target :: test_reqvars1(18) @@ -19,7 +19,6 @@ program test 'banana_array_dim ', & 'cloud_ice_dry_mixing_ratio ', & 'cloud_liquid_dry_mixing_ratio ', & - 'tendency_of_cloud_liquid_dry_mixing_ratio', & 'surface_air_pressure ', & 'temperature ', & 'time_step_for_physics ', & diff --git a/end-to-end-tests/advection/test_host.F90 b/end-to-end-tests/advection/test_host.F90 new file mode 100644 index 00000000..dfbd3c54 --- /dev/null +++ b/end-to-end-tests/advection/test_host.F90 @@ -0,0 +1,1172 @@ +module test_prog + + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + implicit none + private + + public test_host + + ! Public data and interfaces + integer, public, parameter :: cs = 16 + integer, public, parameter :: cm = 41 + + !> \section arg_table_suite_info Argument Table + !! \htmlinclude arg_table_suite_info.html + !! + type, public :: suite_info + character(len=cs) :: suite_name = '' + character(len=cs), pointer :: suite_parts(:) => null() + character(len=cm), pointer :: suite_input_vars(:) => null() + character(len=cm), pointer :: suite_output_vars(:) => null() + character(len=cm), pointer :: suite_required_vars(:) => null() + end type suite_info + + type(ccpp_constituent_properties_t), private, target, allocatable :: host_constituents(:) + + private :: check_suite + private :: advect_constituents ! Move data around + private :: check_errcode + +contains + + subroutine check_errcode(subname, errcode, errmsg, errcode_final) + ! If errcode is not zero, print an error message + character(len=*), intent(in) :: subname + integer, intent(in) :: errcode + character(len=*), intent(in) :: errmsg + + integer, intent(out) :: errcode_final + + if (errcode /= 0) then + write(6, '(a,i0,4a)') "Error ", errcode, " from ", trim(subname), & + ':', trim(errmsg) + !Notify test script that a failure occurred: + errcode_final = -1 !Notify test script that a failure occured + end if + + end subroutine check_errcode + + logical function check_suite(test_suite) + use test_host_ccpp_cap, only: ccpp_physics_suite_part_list + use test_host_ccpp_cap, only: ccpp_physics_suite_variables + use test_utils, only: check_list + + ! Dummy argument + type(suite_info), intent(in) :: test_suite + ! Local variables + logical :: check + integer :: errcode + character(len=512) :: errmsg + character(len=128), allocatable :: test_list(:) + + check_suite = .true. + ! First, check the suite parts + call ccpp_physics_suite_part_list(test_suite%suite_name, test_list, & + errmsg, errcode) + if (errcode == 0) then + check = check_list(test_list, test_suite%suite_parts, 'part names', & + suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errcode, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check the input variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errcode, input_vars=.true., output_vars=.false.) + if (errcode == 0) then + check = check_list(test_list, test_suite%suite_input_vars, & + 'input variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errcode, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check the output variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errcode, input_vars=.false., output_vars=.true.) + if (errcode == 0) then + check = check_list(test_list, test_suite%suite_output_vars, & + 'output variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errcode, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check all required variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errcode) + if (errcode == 0) then + check = check_list(test_list, test_suite%suite_required_vars, & + 'required variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errcode, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + end function check_suite + + subroutine advect_constituents() + use test_host_mod, only: phys_state, & + ncnst + use test_host_mod, only: twist_array + + ! Local variables + integer :: q_ind ! Constituent index + + do q_ind = 1, ncnst ! Skip checks, they were done in constituents_in + call twist_array(phys_state%q(:, :, q_ind)) + end do + end subroutine advect_constituents + + !> \section arg_table_test_host Argument Table + !! \htmlinclude arg_table_test_host.html + !! + subroutine test_host(retval, test_suites) + + use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t + use test_host_mod, only: num_time_steps + use test_host_mod, only: init_data, & + compare_data + use test_host_mod, only: ncols, & + pver + use test_host_data, only: num_consts, & + std_name_array, & + const_std_name + use test_host_data, only: check_constituent_indices + use test_host_ccpp_cap, only: ccpp_deallocate_dynamic_constituents + use test_host_ccpp_cap, only: ccpp_register_constituents + use test_host_ccpp_cap, only: ccpp_is_scheme_constituent + use test_host_ccpp_cap, only: ccpp_initialize_constituents + use test_host_ccpp_cap, only: ccpp_number_constituents + use test_host_ccpp_cap, only: ccpp_constituents_array + use test_host_ccpp_cap, only: ccpp_register + use test_host_ccpp_cap, only: ccpp_init + use test_host_ccpp_cap, only: ccpp_physics_init + use test_host_ccpp_cap, only: ccpp_physics_timestep_init + use test_host_ccpp_cap, only: ccpp_physics_run + use test_host_ccpp_cap, only: ccpp_physics_timestep_final + use test_host_ccpp_cap, only: ccpp_physics_final + use test_host_ccpp_cap, only: ccpp_final + use test_host_ccpp_cap, only: ccpp_physics_suite_list + use test_host_ccpp_cap, only: ccpp_const_get_index + use test_host_ccpp_cap, only: ccpp_model_const_properties + use test_utils, only: check_list + + type(suite_info), intent(in) :: test_suites(:) + logical, intent(out) :: retval + + logical :: check + integer :: col_start, col_end + integer :: index, sind + integer :: index_liq, index_ice + integer :: index_dyn1, index_dyn2, index_dyn3 + integer :: time_step + integer :: num_suites + integer :: num_advected ! Num advected species + logical :: const_log + logical :: is_constituent + logical :: has_default + integer :: test_scalar_const_index + integer :: test_const_indices(num_consts) + character(len=128), allocatable :: suite_names(:) + character(len=256) :: const_str + character(len=512) :: errmsg + character(len=512) :: expected_error + integer :: errcode + integer :: errcode_final ! Used to notify testing script of test failure + real(kind=kind_phys), pointer :: const_ptr(:, :, :) + real(kind=kind_phys) :: default_value + real(kind=kind_phys) :: check_value + type(ccpp_constituent_prop_ptr_t), pointer :: const_props(:) + character(len=*), parameter :: subname = 'test_host' + + ! Initialized "final" error flag used to report a failure to the larged + ! testing script: + errcode_final = 0 + + ! Gather and test the inspection routines + num_suites = size(test_suites) + call ccpp_physics_suite_list(suite_names) + retval = check_list(suite_names, test_suites(:)%suite_name, & + 'suite names') + write(6, *) 'Available suites are:' + do index = 1, size(suite_names) + do sind = 1, num_suites + if (trim(test_suites(sind)%suite_name) == & + trim(suite_names(index))) then + exit + end if + end do + write(6, '(i0,3a,i0,a)') index, ') ', trim(suite_names(index)), & + ' = test_suites(', sind, ')' + end do + if (retval) then + do sind = 1, num_suites + check = check_suite(test_suites(sind)) + retval = retval .and. check + end do + end if + !!! Return here if any check failed + if (.not. retval) then + return + end if + + errcode = 0 + errmsg = '' + + ! Check that is_scheme_constituent works as expected + call ccpp_is_scheme_constituent('specific_humidity', & + is_constituent, errcode, errmsg) + call check_errcode(subname // "_ccpp_is_scheme_constituent", errcode, & + errmsg, errcode_final) + ! specific_humidity should not be an existing constituent + if (is_constituent) then + write(6, *) "ERROR: specific humidity is already a constituent" + errcode_final = -1 ! Notify test script that a failure occurred + end if + call ccpp_is_scheme_constituent('cloud_ice_dry_mixing_ratio', & + is_constituent, errcode, errmsg) + call check_errcode(subname // "_ccpp_is_scheme_constituent", errcode, & + errmsg, errcode_final) + ! cloud_ice_dry_mixing_ratio should be an existing constituent + if (.not. is_constituent) then + write(6, *) "ERROR: cloud_ice_dry_mixing ratio not found in ", & + "host cap constituent list" + errcode_final = -1 ! Notify test script that a failure occurred + end if + + ! Use the suite information to call the register phase + do sind = 1, num_suites + if (errcode == 0) then + call ccpp_register(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(6, '(4a)') 'ERROR in register of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + exit + end if + end if + end do + + ! Register the constituents to find out what needs advecting + ! DO A COUPLE OF TESTS FIRST + + ! First confirm the correct error occurs if you try to add an + ! incompatible constituent with the same standard name + expected_error = 'ccp_model_const_add_metadata ERROR: Trying to add ' //& + 'constituent specific_humidity but an incompatible ' // & + 'constituent with this name already exists' + allocate(host_constituents(2)) + call host_constituents(1)%instantiate(std_name="specific_humidity", & + long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & + vertical_dim="vertical_layer_dimension", advected=.true., & + min_value=1000._kind_phys, molar_mass=2000._kind_phys, & + errcode=errcode, errmsg=errmsg) + call host_constituents(2)%instantiate(std_name="specific_humidity", & + long_name="Specific humidity", diag_name='H2O', units="kg kg", & + vertical_dim="vertical_layer_dimension", advected=.true., & + min_value=1000._kind_phys, molar_mass=2000._kind_phys, & + errcode=errcode, errmsg=errmsg) + call check_errcode(subname // '.initialize', errcode, errmsg, errcode_final) + if (errcode == 0) then + call ccpp_register_constituents(host_constituents, & + errmsg=errmsg, errcode=errcode) + end if + ! Check the error + if (errcode == 0) then + write(6, '(2a)') 'ERROR register_constituents: expected this error: ', & + trim(expected_error) + else + if (trim(errmsg) /= trim(expected_error)) then + write(6, '(4a)') 'ERROR register_constituents: expected this error: ', & + trim(expected_error), ' Got: ', trim(errmsg) + end if + end if + + ! Now try again but with a compatible constituent - should be ignored when + ! the constituents object is created + ! Use the suite information to call the register phase + errcode = 0 + call ccpp_deallocate_dynamic_constituents() + deallocate(host_constituents) + do sind = 1, num_suites + if (errcode == 0) then + call ccpp_register(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(6, '(4a)') 'ERROR in register of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + exit + end if + end if + end do + allocate(host_constituents(3)) + call host_constituents(1)%instantiate(std_name="specific_humidity", & + long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & + vertical_dim="vertical_layer_dimension", advected=.true., & + min_value=1000._kind_phys, molar_mass=2000._kind_phys, & + errcode=errcode, errmsg=errmsg) + call host_constituents(2)%instantiate(std_name="specific_humidity", & + long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & + vertical_dim="vertical_layer_dimension", advected=.true., & + min_value=1000._kind_phys, molar_mass=2000._kind_phys, & + errcode=errcode, errmsg=errmsg) + call host_constituents(3)%instantiate( & + std_name='cloud_ice_dry_mixing_ratio', & + long_name='Cloud ice dry mixing ratio', & + diag_name='CLDICE', & + units='kg kg-1', & + vertical_dim='vertical_layer_dimension', & + advected=.true., & + default_value=0._kind_phys, & + !water_species=.true., & + mixing_ratio_type='dry', & + errcode=errcode, errmsg=errmsg) + + call check_errcode(subname // '.initialize', errcode, errmsg, errcode_final) + if (errcode == 0) then + call ccpp_register_constituents(host_constituents, & + errmsg=errmsg, errcode=errcode) + end if + if (errcode /= 0) then + write(6, '(2a)') 'ERROR register_constituents: ', trim(errmsg) + retval = .false. + return + end if + ! Check number of advected constituents + if (errcode == 0) then + call ccpp_number_constituents(num_advected, errmsg=errmsg, & + errcode=errcode) + call check_errcode(subname // ".num_advected", errcode, errmsg, errcode_final) + end if + if (num_advected /= 6) then + write(6, '(a,i0)') "ERROR: num advected constituents = ", num_advected + retval = .false. + return + end if + ! Initialize constituent data + call ccpp_initialize_constituents(ncols=ncols, num_layers=pver, errcode=errcode, errmsg=errmsg) + + ! Stop tests here if initialization failed (as all other tests will likely + ! fail as well: + if (errcode /= 0) then + retval = .false. + return + end if + + ! Initialize our 'data' + const_ptr => ccpp_constituents_array() + + ! Check if the specific humidity index can be found: + call ccpp_const_get_index('specific_humidity', const_index=index, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_specific_humidity", errcode, errmsg, & + errcode_final) + + ! Check if the cloud liquid index can be found: + call ccpp_const_get_index(stdname='cloud_liquid_dry_mixing_ratio', & + const_index=index_liq, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_cld_liq", errcode, errmsg, & + errcode_final) + + ! Check if the cloud ice index can be found: + call ccpp_const_get_index(stdname='cloud_ice_dry_mixing_ratio', & + const_index=index_ice, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_cld_ice", errcode, errmsg, & + errcode_final) + + ! Check if the dynamic constituents indices can be found + call ccpp_const_get_index(stdname='dyn_const1', const_index=index_dyn1, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_dyn_const1", errcode, errmsg, & + errcode_final) + call ccpp_const_get_index(stdname='dyn_const2_wrt_moist_air', const_index=index_dyn2, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_dyn_const2", errcode, errmsg, & + errcode_final) + call ccpp_const_get_index(stdname='dyn_const3_wrt_moist_air_and_condensed_water', const_index=index_dyn3, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_dyn_const3", errcode, errmsg, & + errcode_final) + + ! Load up the test array indices + call ccpp_const_get_index(stdname=const_std_name, const_index=test_scalar_const_index, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // "." // const_std_name, errcode, errmsg, & + errcode_final) + do sind = 1, num_consts + call ccpp_const_get_index(stdname=std_name_array(sind), & + const_index=test_const_indices(sind), errcode=errcode, errmsg=errmsg) + call check_errcode(subname // "." // std_name_array(sind), errcode, errmsg, & + errcode_final) + end do + + ! Stop tests here if the index checks failed, as all other tests will + ! likely fail as well: + if (errcode_final /= 0) then + retval = .false. + return + end if + + call init_data(const_ptr, index, index_liq, index_ice, index_dyn3) + + ! Check some constituent properties + ! ++++++++++++++++++++++++++++++++++ + + const_props => ccpp_model_const_properties() + + ! Standard name: + call const_props(index)%standard_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get standard_name for specific_humidity, index = ", & + index, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occured + end if + if (errcode == 0) then + if (trim(const_str) /= 'specific_humidity') then + write(6, *) "ERROR: standard name, '", trim(const_str), & + "' should be 'specific_humidity'" + errcode_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + ! Check standard name for a dynamic constituent + call const_props(index_dyn2)%standard_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get standard_name for dyn_const2, index = ", & + index_dyn2, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occured + end if + if (errcode == 0) then + if (trim(const_str) /= 'dyn_const2_wrt_moist_air') then + write(6, *) "ERROR: standard name, '", trim(const_str), & + "' should be 'dyn_const2_wrt_moist_air'" + errcode_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + + ! Long name: + call const_props(index_liq)%long_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get long_name for cld_liq index = ", & + index_liq, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occured + end if + if (errcode == 0) then + if (trim(const_str) /= 'Cloud liquid dry mixing ratio') then + write(6, *) "ERROR: long name, '", trim(const_str), & + "' should be 'Cloud liquid dry mixing ratio'" + errcode_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + ! Check long name for a dynamic constituent + call const_props(index_dyn1)%long_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get long_name for dyn_const1 index = ", & + index_dyn1, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occured + end if + if (errcode == 0) then + if (trim(const_str) /= 'dyn const1') then + write(6, *) "ERROR: long name, '", trim(const_str), & + "' should be 'dyn const1'" + errcode_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + + ! Diagnostic name: + call const_props(index_liq)%diagnostic_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get diagnostic name for cld_liq index = ", & + index_liq, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occured + end if + if (errcode == 0) then + if (trim(const_str) /= 'CLDLIQ') then + write(6, *) "ERROR: diagnostic name, '", trim(const_str), & + "' should be 'CLDLIQ'" + errcode_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + ! Check default diagnostic name is set correctly + call const_props(index_ice)%diagnostic_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get diagnostic name for cld_ice index = ", & + index_ice, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occured + end if + if (errcode == 0) then + if (trim(const_str) /= 'CLDICE') then + write(6, *) "ERROR: diagnostic name, '", trim(const_str), & + "' should be 'CLDICE'" + errcode_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + ! Check diagnostic name of a dynamic constituent + call const_props(index_dyn2)%diagnostic_name(const_str, errcode, & + errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get diagnostic name for dyn_const2 index = ", & + index_dyn2, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occured + end if + if (errcode == 0) then + if (trim(const_str) /= 'DYNCONST2') then + write(6, *) "ERROR: diagnostic name, '", trim(const_str), & + "' should be 'DYNCONST2'" + errcode_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + + ! Mass mixing ratio: + call const_props(index_ice)%is_mass_mixing_ratio(const_log, errcode, & + errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get mass mixing ratio prop for cld_ice index = ", & + index_ice, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occured + end if + if (errcode == 0) then + if (.not. const_log) then + write(6, *) "ERROR: cloud ice is not a mass mixing_ratio" + errcode_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + ! Check mass mixing ratio for a dynamic constituent + call const_props(index_dyn2)%is_mass_mixing_ratio(const_log, errcode, & + errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get mass mixing ratio prop for dyn_const2 index = ", & + index_dyn2, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occured + end if + if (errcode == 0) then + if (.not. const_log) then + write(6, *) "ERROR: dyn_const2 is not a mass mixing_ratio" + errcode_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + + ! Dry mixing ratio: + call const_props(index_ice)%is_dry(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get dry prop for cld_ice index = ", index_ice, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + if (errcode == 0) then + if (.not. const_log) then + write(6, *) "ERROR: cloud ice mass_mixing_ratio is not dry" + errcode_final = -1 + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + ! Check wet mixing ratio for dynamic constituent 1 + call const_props(index_dyn1)%is_dry(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get dry prop for dyn_const1 index = ", index_dyn1, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + if (errcode == 0) then + if (const_log) then + write(6, *) "ERROR: dyn_const1 is dry and should be wet" + errcode_final = -1 + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + call const_props(index_dyn1)%is_wet(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get wet prop for dyn_const1 index = ", index_dyn1, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + if (errcode == 0) then + if (.not. const_log) then + write(6, *) "ERROR: dyn_const1 is not wet but should be" + errcode_final = -1 + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + ! Check moist mixing ratio for dynamic constituent 2 + call const_props(index_dyn2)%is_dry(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get dry prop for dyn_const2 index = ", index_dyn2, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + if (errcode == 0) then + if (const_log) then + write(6, *) "ERROR: dyn_const2 is dry and should be moist" + errcode_final = -1 + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + call const_props(index_dyn2)%is_moist(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get moist prop for dyn_const2 index = ", index_dyn2, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + if (errcode == 0) then + if (.not. const_log) then + write(6, *) "ERROR: dyn_const2 is not moist but should be" + errcode_final = -1 + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + ! Check dry mixing ratio for dynamic constituent 3 + call const_props(index_dyn3)%is_dry(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get dry prop for dyn_const3 index = ", index_dyn3, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + if (errcode == 0) then + if (.not. const_log) then + write(6, *) "ERROR: dyn_const3 is not dry and should be" + errcode_final = -1 + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + + ! ------------------- + + ! ------------------- + ! minimum value tests: + ! ------------------- + + ! Check that a constituent's minimum value defaults to zero: + call const_props(index_dyn2)%minimum(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get minimum value for dyn_const2 index = ", index_dyn2, & + trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + if (errcode == 0) then + if (check_value /= 0._kind_phys) then ! Should be zero + write(6, *) "ERROR: 'minimum' should default to zero for all ", & + "constituents unless set by host model or scheme metadata." + errcode_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + + ! Check that a constituent instantiated with a specified minimum value + ! actually contains that minimum value property: + call const_props(index_dyn1)%minimum(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get minimum value for dyn_const1 index = ", index_dyn1, & + trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + if (errcode == 0) then + if (check_value /= 1000._kind_phys) then !Should be 1000 + write(6, *) "ERROR: 'minimum' should give a value of 1000 ", & + "for dyn_const1, as was set during instantiation." + errcode_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + + ! Check that setting a constituent's minimum value works + ! as expected: + call const_props(index_dyn1)%set_minimum(1._kind_phys, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to set minimum value for dyn_const1 index = ", index_dyn1, & + trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + if (errcode == 0) then + call const_props(index_dyn1)%minimum(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & + " trying to get minimum value for dyn_const1 index = ", & + index_dyn1, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + end if + if (errcode == 0) then + if (check_value /= 1._kind_phys) then ! Should now be one + write(6, *) "ERROR: 'set_minimum' did not set constituent", & + " minimum value correctly." + errcode_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + + ! ---------------------- + ! molecular weight tests: + ! ---------------------- + + ! Check that a constituent instantiated with a specified molecular + ! weight actually contains that molecular weight property value: + call const_props(index)%molar_mass(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get molecular weight for specific humidity index = ", & + index, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + if (errcode == 0) then + if (check_value /= 2000._kind_phys) then ! Should be 2000 + write(6, *) "ERROR: 'molar_mass' should give a value of 2000 ", & + "for specific humidity, as was set during instantiation." + errcode_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + + ! Check that setting a constituent's molecular weight works + ! as expected: + call const_props(index_ice)%set_molar_mass(1._kind_phys, errcode, & + errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to set molecular weight for cld_ice index = ", index_ice, & + trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + if (errcode == 0) then + call const_props(index_ice)%molar_mass(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & + " trying to get molecular weight for cld_ice index = ", & + index_ice, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + end if + if (errcode == 0) then + if (check_value /= 1._kind_phys) then ! Should be equal to one + write(6, *) "ERROR: 'set_molar_mass' did not set constituent", & + " molecular weight value correctly." + errcode_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + + ! ------------------- + ! thermo-active tests: + ! ------------------- + + ! Check that being thermodynamically active defaults to False: + call const_props(index_ice)%is_thermo_active(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get thermo_active prop for cld_ice index = ", index_ice, & + trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + if (errcode == 0) then + if (check) then ! Should be False + write(6, *) "ERROR: 'is_thermo_active' should default to False ", & + "for all constituents unless set by host model." + errcode_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + + ! Check that setting a constituent to be thermodynamically active works + ! as expected: + call const_props(index_ice)%set_thermo_active(.true., errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to set thermo_active prop for cld_ice index = ", index_ice, & + trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + if (errcode == 0) then + call const_props(index_ice)%is_thermo_active(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & + " trying to get thermo_active prop for cld_ice index = ", & + index_ice, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + end if + if (errcode == 0) then + if (.not. check) then ! Should now be True + write(6, *) "ERROR: 'set_thermo_active' did not set", & + " thermo_active constituent property correctly." + errcode_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + ! ------------------- + + ! ------------------- + ! water-species tests: + ! ------------------- + + ! Check that being a water species defaults to False: + call const_props(index_liq)%is_water_species(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to get water_species prop for cld_liq index = ", index_liq, & + trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + if (errcode == 0) then + if (check) then ! Should be False + write(6, *) "ERROR: 'is_water_species' should default to False ", & + "for all constituents unless set by host model." + errcode_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + + ! Check that setting a constituent to be a water species works + ! as expected: + call const_props(index_liq)%set_water_species(.true., errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to set water_species prop for cld_liq index = ", index_liq, & + trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + if (errcode == 0) then + call const_props(index_liq)%is_water_species(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & + " trying to get water_species prop for cld_liq index = ", & + index_liq, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + end if + if (errcode == 0) then + if (.not. check) then ! Should now be True + write(6, *) "ERROR: 'set_water_species' did not set", & + " water_species constituent property correctly." + errcode_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + + ! Check that setting a constituent to be a water species via the + ! instantiate call works as expected + call const_props(index_dyn1)%is_water_species(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & + "trying to get water_species prop for dyn_const1 index = ", & + index_dyn1, trim(errmsg) + end if + if (errcode == 0) then + if (.not. check) then ! Should now be True + write(6, *) "ERROR: 'water_species=.true. did not set", & + " water_species constituent property correctly" + errcode_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + call const_props(index_dyn2)%is_water_species(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & + "trying to get water_species prop for dyn_const2 index = ", & + index_dyn2, trim(errmsg) + end if + if (errcode == 0) then + if (check) then ! Should now be False + write(6, *) "ERROR: 'water_species=.false. did not set", & + " water_species constituent property correctly" + errcode_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + ! ------------------- + + ! Check that setting a constituent's default value works as expected + call const_props(index_liq)%has_default(has_default, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to check for default for cld_liq index = ", index_liq, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + if (errcode == 0) then + if (.not. has_default) then + write(6, *) "ERROR: cloud_liquid_dry_mixing_ratio should have default but doesn't" + errcode_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + call const_props(index_ice)%has_default(has_default, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to check for default for cld_ice index = ", index_ice, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + if (errcode == 0) then + if (.not. has_default) then + write(6, *) "ERROR: cloud ice_dry_mixing_ratio should have default but doesn't" + errcode_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + call const_props(index_ice)%default_value(default_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & + "to grab default for cld_ice index = ", index_ice, trim(errmsg) + errcode_final = -1 ! Notify test script that a failure occurred + end if + if (errcode == 0) then + if (default_value /= 0.0_kind_phys) then + write(6, *) "ERROR: cloud ice mass_mixing_ratio default is ", default_value, & + " but should be 0.0" + errcode_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errcode = 0 + end if + ! ++++++++++++++++++++++++++++++++++ + + ! Set error flag to the "final" value, because any error + ! above will likely result in a large number of failures + ! below: + errcode = errcode_final + + ! Call ccpp_init + do sind = 1, num_suites + if (errcode == 0) then + call ccpp_init(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(6, '(4a)') 'ERROR in initialize of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + exit + end if + end if + end do + + ! Call ccpp_physics_init + do sind = 1, num_suites + if (errcode == 0) then + call ccpp_physics_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(6, '(4a)') 'ERROR in initialize of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + exit + end if + end if + end do + + ! Check indices + call check_constituent_indices(test_scalar_const_index, test_const_indices, & + errmsg, errcode) + call check_errcode(subname // " check suite indices", errcode, errmsg, & + errcode_final) + + ! Loop over time steps + do time_step = 1, num_time_steps + ! Initialize the timestep + do sind = 1, num_suites + if (errcode == 0) then + call ccpp_physics_timestep_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & + trim(errmsg) + end if + end if + end do + + do col_start = 1, ncols, 5 + if (errcode /= 0) then + continue + end if + col_end = min(col_start + 4, ncols) + + do sind = 1, num_suites + do index = 1, size(test_suites(sind)%suite_parts) + if (errcode == 0) then + call ccpp_physics_run( & + suite_name=test_suites(sind)%suite_name, & + group_name=test_suites(sind)%suite_parts(index), & + col_start=col_start, col_end=col_end, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(6, '(5a)') trim(test_suites(sind)%suite_name), & + '/', trim(test_suites(sind)%suite_parts(index)),& + ': ', trim(errmsg) + exit + end if + end if + end do + end do + end do + ! Check indices + call check_constituent_indices(test_scalar_const_index, test_const_indices, & + errmsg, errcode) + call check_errcode(subname // " check suite indices", errcode, errmsg, & + errcode_final) + + do sind = 1, num_suites + if (errcode == 0) then + call ccpp_physics_timestep_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) + end if + if (errcode /= 0) then + write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & + trim(errmsg) + exit + end if + end do + + ! Run "dycore" + if (errcode == 0) then + call advect_constituents() + end if + end do ! End time step loop + + do sind = 1, num_suites + if (errcode == 0) then + call ccpp_physics_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_physics_final, ', & + 'Exiting...' + exit + end if + end if + end do + + do sind = 1, num_suites + if (errcode == 0) then + call ccpp_final(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_final, ', & + 'Exiting...' + exit + end if + end if + end do + + call ccpp_deallocate_dynamic_constituents() + deallocate(host_constituents) + + if (errcode == 0) then + ! Run finished without error, check answers + if (compare_data(num_advected)) then + write(6, *) 'Answers are correct!' + errcode = 0 + else + write(6, *) 'Answers are not correct!' + errcode = -1 + end if + end if + + ! Make sure "final" flag is non-zero if "errcode" is: + if (errcode /= 0) then + errcode_final = -1 ! Notify test script that a failure occured + end if + + ! Set return value to False if any errors were found: + retval = errcode_final == 0 + + end subroutine test_host + +end module test_prog diff --git a/end-to-end-tests/advection/test_host.meta b/end-to-end-tests/advection/test_host.meta new file mode 100644 index 00000000..e69dafd9 --- /dev/null +++ b/end-to-end-tests/advection/test_host.meta @@ -0,0 +1,70 @@ +[ccpp-table-properties] + name = suite_info + type = ddt +[ccpp-arg-table] + name = suite_info + type = ddt + +[ccpp-table-properties] + name = test_host + type = control +[ccpp-arg-table] + name = test_host + type = control +[ col_start ] + standard_name = horizontal_loop_begin + type = integer + units = count + dimensions = () + protected = True +[ col_end ] + standard_name = horizontal_loop_end + type = integer + units = count + dimensions = () + protected = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 +[ errcode ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer diff --git a/test/advection_test/test_host_data.F90 b/end-to-end-tests/advection/test_host_data.F90 similarity index 96% rename from test/advection_test/test_host_data.F90 rename to end-to-end-tests/advection/test_host_data.F90 index f360ad79..4bcb753b 100644 --- a/test/advection_test/test_host_data.F90 +++ b/end-to-end-tests/advection/test_host_data.F90 @@ -29,26 +29,26 @@ module test_host_data contains - subroutine check_constituent_indices(test_index, test_indices, errmsg, errflg) + subroutine check_constituent_indices(test_index, test_indices, errmsg, errcode) ! Check constituent indices against what was found by suite ! indices are passed in rather than looked up to avoid a dependency loop ! Dummy arguments integer, intent(in) :: test_index ! scalar const index from host integer, intent(in) :: test_indices(:) ! array_test_indices from host character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode ! Local variable integer :: indx integer :: emstrt - errflg = 0 + errcode = 0 errmsg = '' if (test_index /= const_index) then emstrt = len_trim(errmsg) + 1 write(errmsg(emstrt:), '(2a,i0,a,i0)') 'const_index_check for ', & const_std_name, test_index, ' /= ', const_index - errflg = errflg + 1 + errcode = errcode + 1 end if do indx = 1, num_consts if (test_indices(indx) /= const_inds(indx)) then @@ -59,7 +59,7 @@ subroutine check_constituent_indices(test_index, test_indices, errmsg, errflg) end if write(errmsg(emstrt:), '(2a,i0,a,i0)') 'const_indices_check for ', & std_name_array(indx), test_indices(indx), ' /= ', const_inds(indx) - errflg = errflg + 1 + errcode = errcode + 1 end if end do diff --git a/test/advection_test/test_host_data.meta b/end-to-end-tests/advection/test_host_data.meta similarity index 94% rename from test/advection_test/test_host_data.meta rename to end-to-end-tests/advection/test_host_data.meta index a676f141..960ce33e 100644 --- a/test/advection_test/test_host_data.meta +++ b/end-to-end-tests/advection/test_host_data.meta @@ -6,7 +6,6 @@ type = ddt [ ps ] standard_name = surface_air_pressure - state_variable = true type = real kind = kind_phys units = Pa @@ -18,14 +17,12 @@ type = real | kind = kind_phys [ q ] standard_name = constituent_mixing_ratio - state_variable = true type = real kind = kind_phys units = kg kg-1 moist or dry air depending on type dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_tracers) [ q(:,:,index_of_water_vapor_specific_humidity) ] standard_name = water_vapor_specific_humidity - state_variable = true type = real kind = kind_phys units = kg kg-1 @@ -33,10 +30,10 @@ [ccpp-table-properties] name = test_host_data - type = module + type = host [ccpp-arg-table] name = test_host_data - type = module + type = host [ num_consts ] standard_name = banana_array_dim long_name = Size of test_banana_name_array diff --git a/test/advection_test/test_host_mod.F90 b/end-to-end-tests/advection/test_host_mod.F90 similarity index 100% rename from test/advection_test/test_host_mod.F90 rename to end-to-end-tests/advection/test_host_mod.F90 diff --git a/test/advection_test/test_host_mod.meta b/end-to-end-tests/advection/test_host_mod.meta similarity index 97% rename from test/advection_test/test_host_mod.meta rename to end-to-end-tests/advection/test_host_mod.meta index 9f04a6fc..6c3d15eb 100644 --- a/test/advection_test/test_host_mod.meta +++ b/end-to-end-tests/advection/test_host_mod.meta @@ -1,9 +1,9 @@ [ccpp-table-properties] name = test_host_mod - type = module + type = host [ccpp-arg-table] name = test_host_mod - type = module + type = host [ ncols] standard_name = horizontal_dimension units = count diff --git a/end-to-end-tests/advection_auto_clone/CMakeLists.txt b/end-to-end-tests/advection_auto_clone/CMakeLists.txt new file mode 100644 index 00000000..30884200 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/CMakeLists.txt @@ -0,0 +1,95 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "cld_liq" "cld_ice" "apply_constituent_tendencies" "const_indices") +set(HOST_FILES "test_host_data" "test_host_mod" "test_host") +set(SUITE_FILES "cld_suite.xml") +set(HOST "test_host") +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Enable legacy flags for the capgen call +set(EXTRA_FLAGS + "--legacy-mode" + "--legacy-auto-clone-constituents" +) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME" + EXTRA_FLAGS ${EXTRA_FLAGS}) +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST" + EXTRA_FLAGS ${EXTRA_FLAGS}) + +# Enable trace output in auto-generated caps +set(CCPP_TRACE ON) + +# Enable legacy flags for the capgen call +set(EXTRA_FLAGS + "--legacy-mode" + "--legacy-auto-clone-constituents" +) + +# Run ccpp_capgen +ccpp_capgen(TRACE ${CCPP_TRACE} + VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + EXTRA_FLAGS ${EXTRA_FLAGS} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +# Add extra files needed for testing +set(EXTRA_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/../utils/test_utils.F90 +) + +add_executable(test_advection_auto_clone.x + ${EXTRA_FILES} + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES_FILTERED} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES} + test_advection_host_integration.F90 +) +target_link_libraries(test_advection_auto_clone.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_advection_auto_clone.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_advection_auto_clone.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_advection_auto_clone + COMMAND test_advection_auto_clone.x) diff --git a/end-to-end-tests/advection_auto_clone/README.md b/end-to-end-tests/advection_auto_clone/README.md new file mode 100644 index 00000000..616b9036 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/README.md @@ -0,0 +1,10 @@ +# Advection Test + +Contains tests to exercise the capabilities of the constituents object, including: +- Adding build-time constituents via metadata property +- Adding run-time constituents from schemes via a register phase + - Also tests that trying to add a constituent outside of the register phase errors as expected +- Passing around and modifying the constituent array +- Accessing and modifying a constituent tendency variable +- Passing around the constituent tendency array +- Dimensions are case-insensitive diff --git a/end-to-end-tests/advection_auto_clone/advection_test_reports.py b/end-to-end-tests/advection_auto_clone/advection_test_reports.py new file mode 100644 index 00000000..4fbe8e68 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/advection_test_reports.py @@ -0,0 +1,127 @@ +#! /usr/bin/env python3 +""" +----------------------------------------------------------------------- + Description: Test advection database report python interface + + Assumptions: + + Command line arguments: build_dir database_filepath + + Usage: python test_reports +----------------------------------------------------------------------- +""" +import os +import unittest + +from test_stub import BaseTests + +_BUILD_DIR = os.path.join(os.path.abspath(os.environ['BUILD_DIR']), "test", "advection_test") +_DATABASE = os.path.abspath(os.path.join(_BUILD_DIR, "ccpp", "datatable.xml")) + +_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +_FRAMEWORK_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, os.pardir)) +_SCRIPTS_DIR = os.path.abspath(os.path.join(_FRAMEWORK_DIR, "scripts")) + +# Check data +_HOST_FILES = [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90")] +_SUITE_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_cld_suite_cap.F90")] +_UTILITY_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_kinds.F90"), + os.path.join(_FRAMEWORK_DIR, "src", + "ccpp_constituent_prop_mod.F90"), + os.path.join(_FRAMEWORK_DIR, "src", + "ccpp_scheme_utils.F90"), + os.path.join(_FRAMEWORK_DIR, "src", "ccpp_hashable.F90"), + os.path.join(_FRAMEWORK_DIR, "src", "ccpp_hash_table.F90")] +_CCPP_FILES = _UTILITY_FILES + _HOST_FILES + _SUITE_FILES +_DEPENDENCIES = [""] +_PROCESS_LIST = [""] +_MODULE_LIST = ["cld_ice", "cld_liq", "const_indices", "apply_constituent_tendencies"] +_SUITE_LIST = ["cld_suite"] +_REQUIRED_VARS_CLD = ["ccpp_error_code", "ccpp_error_message", + "horizontal_loop_begin", "horizontal_loop_end", + "surface_air_pressure", "temperature", + "tendency_of_cloud_liquid_dry_mixing_ratio", + "time_step_for_physics", "water_temperature_at_freezing", + "water_vapor_specific_humidity", + "cloud_ice_dry_mixing_ratio", + "cloud_liquid_dry_mixing_ratio", + "ccpp_constituents", + "ccpp_constituent_tendencies", + "number_of_ccpp_constituents", + "dynamic_constituents_for_cld_ice", + "dynamic_constituents_for_cld_liq", + "test_banana_constituent_indices", "test_banana_name", + "banana_array_dim", + "test_banana_name_array", + "test_banana_constituent_index", + # Added by --debug option + "horizontal_dimension", + "vertical_layer_dimension"] +_INPUT_VARS_CLD = ["surface_air_pressure", "temperature", + "horizontal_loop_begin", "horizontal_loop_end", + "time_step_for_physics", "water_temperature_at_freezing", + "water_vapor_specific_humidity", + "cloud_ice_dry_mixing_ratio", + "cloud_liquid_dry_mixing_ratio", + "tendency_of_cloud_liquid_dry_mixing_ratio", + "ccpp_constituents", + "ccpp_constituent_tendencies", + "number_of_ccpp_constituents", + "banana_array_dim", + "test_banana_name_array", "test_banana_name", + # Added by --debug option + "horizontal_dimension", + "vertical_layer_dimension"] +_OUTPUT_VARS_CLD = ["ccpp_error_code", "ccpp_error_message", + "water_vapor_specific_humidity", "temperature", + "tendency_of_cloud_liquid_dry_mixing_ratio", + "cloud_ice_dry_mixing_ratio", + "ccpp_constituents", + "ccpp_constituent_tendencies", + "cloud_liquid_dry_mixing_ratio", + "dynamic_constituents_for_cld_ice", + "dynamic_constituents_for_cld_liq", + "dynamic_constituents_for_cld_liq", + "test_banana_constituent_indices", + "test_banana_constituent_index"] + + +class TestAdvectionHostDataTables(unittest.TestCase, BaseTests.TestHostDataTables): + database = _DATABASE + host_files = _HOST_FILES + suite_files = _SUITE_FILES + utility_files = _UTILITY_FILES + ccpp_files = _CCPP_FILES + process_list = _PROCESS_LIST + module_list = _MODULE_LIST + dependencies = _DEPENDENCIES + suite_list = _SUITE_LIST + +class CommandLineAdvectionHostDatafileRequiredFiles(unittest.TestCase, BaseTests.TestHostCommandLineDataFiles): + database = _DATABASE + host_files = _HOST_FILES + suite_files = _SUITE_FILES + utility_files = _UTILITY_FILES + ccpp_files = _CCPP_FILES + process_list = _PROCESS_LIST + module_list = _MODULE_LIST + dependencies = _DEPENDENCIES + suite_list = _SUITE_LIST + datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" + + +class TestCapgenCldSuite(unittest.TestCase, BaseTests.TestSuite): + database = _DATABASE + required_vars = _REQUIRED_VARS_CLD + input_vars = _INPUT_VARS_CLD + output_vars = _OUTPUT_VARS_CLD + suite_name = "cld_suite" + + +class CommandLineCapgenDdtSuite(unittest.TestCase, BaseTests.TestSuiteCommandLine): + database = _DATABASE + required_vars = _REQUIRED_VARS_CLD + input_vars = _INPUT_VARS_CLD + output_vars = _OUTPUT_VARS_CLD + suite_name = "cld_suite" + datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" diff --git a/end-to-end-tests/advection_auto_clone/apply_constituent_tendencies.F90 b/end-to-end-tests/advection_auto_clone/apply_constituent_tendencies.F90 new file mode 100644 index 00000000..63a1881c --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/apply_constituent_tendencies.F90 @@ -0,0 +1,39 @@ +module apply_constituent_tendencies + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: apply_constituent_tendencies_run + +contains + + !> \section arg_table_apply_constituent_tendencies_run Argument Table + !!! \htmlinclude apply_constituent_tendencies_run.html + subroutine apply_constituent_tendencies_run(const_tend, const, errcode, errmsg) + ! Dummy arguments + real(kind=kind_phys), intent(inout) :: const_tend(:, :, :) ! constituent tendency array + real(kind=kind_phys), intent(inout) :: const(:, :, :) ! constituent state array + integer, intent(out) :: errcode + character(len=512), intent(out) :: errmsg + + ! Local variables + integer :: klev, jcnst, icol + + errcode = 0 + errmsg = '' + + do icol = 1, size(const_tend, 1) + do klev = 1, size(const_tend, 2) + do jcnst = 1, size(const_tend, 3) + const(icol, klev, jcnst) = const(icol, klev, jcnst) + const_tend(icol, klev, jcnst) + end do + end do + end do + + const_tend = 0._kind_phys + + end subroutine apply_constituent_tendencies_run + +end module apply_constituent_tendencies diff --git a/end-to-end-tests/advection_auto_clone/apply_constituent_tendencies.meta b/end-to-end-tests/advection_auto_clone/apply_constituent_tendencies.meta new file mode 100644 index 00000000..ac02e5e4 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/apply_constituent_tendencies.meta @@ -0,0 +1,36 @@ +##################################################################### +[ccpp-table-properties] + name = apply_constituent_tendencies + type = scheme +[ccpp-arg-table] + name = apply_constituent_tendencies_run + type = scheme +[ const_tend ] + standard_name = ccpp_constituent_tendencies + long_name = ccpp constituent tendencies + units = none + type = real | kind = kind_phys + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) + intent = inout +[ const ] + standard_name = ccpp_constituents + long_name = ccpp constituents + units = none + type = real | kind = kind_phys + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) + intent = inout +[ errcode ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + type = integer + dimensions = () + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + type = character | kind = len=512 + dimensions = () + intent = out +######################################################### diff --git a/end-to-end-tests/advection_auto_clone/cld_ice.F90 b/end-to-end-tests/advection_auto_clone/cld_ice.F90 new file mode 100644 index 00000000..e3fc2abd --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/cld_ice.F90 @@ -0,0 +1,125 @@ +! Test parameterization with advected species +! + +module cld_ice + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: cld_ice_register + public :: cld_ice_init + public :: cld_ice_run + public :: cld_ice_final + + real(kind=kind_phys), private :: tcld = huge(1.0_kind_phys) + +contains + + !> \section arg_table_cld_ice_register Argument Table + !! \htmlinclude arg_table_cld_ice_register.html + !! + subroutine cld_ice_register(dyn_const_ice, errmsg, errcode) + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const_ice(:) + integer, intent(out) :: errcode + character(len=512), intent(out) :: errmsg + + errmsg = '' + errcode = 0 + allocate(dyn_const_ice(2), stat=errcode) + if (errcode /= 0) then + errmsg = 'Error allocating dyn_const in cld_ice_dynamic_constituents' + return + end if + call dyn_const_ice(1)%instantiate(std_name='dyn_const1', long_name='dyn const1', & + diag_name='DYNCONST1', units='kg kg-1', default_value=0._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + min_value=1000._kind_phys, water_species=.true., mixing_ratio_type='wet', & + errcode=errcode, errmsg=errmsg) + call dyn_const_ice(2)%instantiate(std_name='dyn_const2_wrt_moist_air', long_name='dyn const2', & + diag_name='DYNCONST2', units='kg kg-1', default_value=0._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + water_species=.false., errcode=errcode, errmsg=errmsg) + + end subroutine cld_ice_register + + !> \section arg_table_cld_ice_run Argument Table + !! \htmlinclude arg_table_cld_ice_run.html + !! + subroutine cld_ice_run(ncol, timestep, temp, qv, ps, cld_ice_array, & + errmsg, errcode) + + integer, intent(in) :: ncol + real(kind=kind_phys), intent(in) :: timestep + real(kind=kind_phys), intent(inout) :: temp(:, :) + real(kind=kind_phys), intent(inout) :: qv(:, :) + real(kind=kind_phys), intent(in) :: ps(:) + real(kind=kind_phys), intent(inout) :: cld_ice_array(:, :) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errcode + !---------------------------------------------------------------- + + integer :: icol + integer :: ilev + real(kind=kind_phys) :: frz + + errmsg = '' + errcode = 0 + + ! Apply state-of-the-art thermodynamics :) + do icol = 1, ncol + do ilev = 1, size(temp, 2) + if (temp(icol, ilev) < tcld) then + frz = max(qv(icol, ilev) - 0.5_kind_phys, 0.0_kind_phys) + cld_ice_array(icol, ilev) = cld_ice_array(icol, ilev) + frz + qv(icol, ilev) = qv(icol, ilev) - frz + if (frz > 0.0_kind_phys) then + temp(icol, ilev) = temp(icol, ilev) + 1.0_kind_phys + end if + end if + end do + end do + + end subroutine cld_ice_run + + !> \section arg_table_cld_ice_init Argument Table + !! \htmlinclude arg_table_cld_ice_init.html + !! + subroutine cld_ice_init(tfreeze, errmsg, errcode) + + real(kind=kind_phys), intent(in) :: tfreeze + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errcode + + errmsg = '' + errcode = 0 + tcld = tfreeze - 20.0_kind_phys + + end subroutine cld_ice_init + + !> \section arg_table_cld_ice_final Argument Table + !! \htmlinclude arg_table_cld_ice_final.html + !! + + !> @{ + !! This routine does nothing, but it tests if blank + !! lines and doxygen comments between metadata hooks + !! and the subroutine are parsed correctly. + !! @{ + + subroutine cld_ice_final(errmsg, errcode) + + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errcode + + errmsg = '' + errcode = 0 + + end subroutine cld_ice_final + + !! @} + !! @} + +end module cld_ice diff --git a/test/advection_test/cld_ice.meta b/end-to-end-tests/advection_auto_clone/cld_ice.meta similarity index 83% rename from test/advection_test/cld_ice.meta rename to end-to-end-tests/advection_auto_clone/cld_ice.meta index e57d0b08..7496da74 100644 --- a/test/advection_test/cld_ice.meta +++ b/end-to-end-tests/advection_auto_clone/cld_ice.meta @@ -2,6 +2,7 @@ [ccpp-table-properties] name = cld_ice type = scheme + [ccpp-arg-table] name = cld_ice_register type = scheme @@ -27,11 +28,12 @@ dimensions = () type = integer intent = out + [ccpp-arg-table] name = cld_ice_run type = scheme [ ncol ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension type = integer units = count dimensions = () @@ -47,31 +49,30 @@ [ temp ] standard_name = temperature units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = inout [ qv ] standard_name = water_vapor_specific_humidity units = kg kg-1 - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = inout [ ps ] standard_name = surface_air_pressure - state_variable = true type = real kind = kind_phys units = Pa - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) intent = in [ cld_ice_array ] standard_name = cloud_ice_dry_mixing_ratio advected = .true. default_value = 0.0_kind_phys units = kg kg-1 - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real | kind = kind_phys intent = inout [ errmsg ] @@ -82,13 +83,14 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 dimensions = () type = integer intent = out + [ccpp-arg-table] name = cld_ice_init type = scheme @@ -99,15 +101,6 @@ dimensions = () type = real | kind = kind_phys intent = in -[ cld_ice_array ] - standard_name = cloud_ice_dry_mixing_ratio - advected = .true. - default_value = 0.0_kind_phys - units = kg kg-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) - type = real | kind = kind_phys - # Advected species that needs to be supplied by framework - intent = inout [ errmsg ] standard_name = ccpp_error_message long_name = Error message for error handling in CCPP @@ -116,13 +109,14 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 dimensions = () type = integer intent = out + [ccpp-arg-table] name = cld_ice_final type = scheme @@ -134,7 +128,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 diff --git a/test/advection_test/cld_liq.F90 b/end-to-end-tests/advection_auto_clone/cld_liq.F90 similarity index 84% rename from test/advection_test/cld_liq.F90 rename to end-to-end-tests/advection_auto_clone/cld_liq.F90 index cb02cf11..586a68b3 100644 --- a/test/advection_test/cld_liq.F90 +++ b/end-to-end-tests/advection_auto_clone/cld_liq.F90 @@ -18,15 +18,15 @@ module cld_liq !> \section arg_table_cld_liq_register Argument Table !! \htmlinclude arg_table_cld_liq_register.html !! - subroutine cld_liq_register(dyn_const, errmsg, errflg) + subroutine cld_liq_register(dyn_const, errmsg, errcode) type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode errmsg = '' - errflg = 0 - allocate(dyn_const(1), stat=errflg) - if (errflg /= 0) then + errcode = 0 + allocate(dyn_const(1), stat=errcode) + if (errcode /= 0) then errmsg = 'Error allocating dyn_const in cld_liq_register' return end if @@ -34,7 +34,7 @@ subroutine cld_liq_register(dyn_const, errmsg, errflg) diag_name='DYNCONST3', units='kg kg-1', default_value=1._kind_phys, & vertical_dim='vertical_layer_dimension', advected=.true., & water_species=.true., mixing_ratio_type='dry', & - errcode=errflg, errmsg=errmsg) + errcode=errcode, errmsg=errmsg) end subroutine cld_liq_register @@ -42,7 +42,7 @@ end subroutine cld_liq_register !! \htmlinclude arg_table_cld_liq_run.html !! subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, & - cld_liq_tend, errmsg, errflg) + cld_liq_tend, errmsg, errcode) integer, intent(in) :: ncol real(kind=kind_phys), intent(in) :: timestep @@ -50,9 +50,9 @@ subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, & real(kind=kind_phys), intent(inout) :: temp(:, :) real(kind=kind_phys), intent(inout) :: qv(:, :) real(kind=kind_phys), intent(in) :: ps(:) - real(kind=kind_phys), intent(inout) :: cld_liq_tend(:, :) + real(kind=kind_phys), intent(out) :: cld_liq_tend(:, :) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode !---------------------------------------------------------------- integer :: icol @@ -60,7 +60,7 @@ subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, & real(kind=kind_phys) :: cond errmsg = '' - errflg = 0 + errcode = 0 ! Apply state-of-the-art thermodynamics :) do icol = 1, ncol @@ -82,18 +82,18 @@ end subroutine cld_liq_run !> \section arg_table_cld_liq_init Argument Table !! \htmlinclude arg_table_cld_liq_init.html !! - subroutine cld_liq_init(tfreeze, cld_liq_array, tcld, errmsg, errflg) + subroutine cld_liq_init(tfreeze, cld_liq_array, tcld, errmsg, errcode) real(kind=kind_phys), intent(in) :: tfreeze - real(kind=kind_phys), intent(out) :: cld_liq_array(:, :) + real(kind=kind_phys), intent(inout) :: cld_liq_array(:, :) real(kind=kind_phys), intent(out) :: tcld character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode ! This routine currently does nothing errmsg = '' - errflg = 0 + errcode = 0 cld_liq_array = 0.0_kind_phys tcld = tfreeze - 20.0_kind_phys diff --git a/test/advection_test/cld_liq.meta b/end-to-end-tests/advection_auto_clone/cld_liq.meta similarity index 88% rename from test/advection_test/cld_liq.meta rename to end-to-end-tests/advection_auto_clone/cld_liq.meta index b3ef3a0d..a26e5600 100644 --- a/test/advection_test/cld_liq.meta +++ b/end-to-end-tests/advection_auto_clone/cld_liq.meta @@ -19,18 +19,19 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 dimensions = () type = integer intent = out + [ccpp-arg-table] name = cld_liq_run type = scheme [ ncol ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension type = integer units = count dimensions = () @@ -52,31 +53,30 @@ [ temp ] standard_name = temperature units = K - dimensions = (horizontal_loop_extent, vertical_LAYER_dimension) + dimensions = (horizontal_dimension, vertical_LAYER_dimension) type = real kind = kind_phys intent = inout [ qv ] standard_name = water_vapor_specific_humidity units = kg kg-1 - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = inout [ ps ] standard_name = surface_air_pressure - state_variable = true type = real kind = kind_phys units = hPa - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) intent = in [ cld_liq_tend ] standard_name = tendency_of_cloud_liquid_dry_mixing_ratio units = kg kg-1 s-1 - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real | kind = kind_phys - intent = inout + intent = out constituent = True [ errmsg ] standard_name = ccpp_error_message @@ -86,13 +86,14 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 dimensions = () type = integer intent = out + [ccpp-arg-table] name = cld_liq_init type = scheme @@ -111,7 +112,8 @@ dimensions = (horizontal_dimension, vertical_layer_dimension) type = real | kind = kind_phys # Advected species that needs to be promoted from suite. - intent = out + # Note that in capgen, intent 'out' is no longer permitted + intent = inout [ tcld] standard_name = minimum_temperature_for_cloud_liquid units = K @@ -126,7 +128,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 diff --git a/end-to-end-tests/advection_auto_clone/cld_suite.xml b/end-to-end-tests/advection_auto_clone/cld_suite.xml new file mode 100644 index 00000000..fac613e8 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/cld_suite.xml @@ -0,0 +1,11 @@ + + + + + const_indices + cld_liq + apply_constituent_tendencies + cld_ice + apply_constituent_tendencies + + diff --git a/end-to-end-tests/advection_auto_clone/cld_suite_error.xml b/end-to-end-tests/advection_auto_clone/cld_suite_error.xml new file mode 100644 index 00000000..80acac91 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/cld_suite_error.xml @@ -0,0 +1,9 @@ + + + + + dlc_liq + cld_liq + cld_ice + + diff --git a/end-to-end-tests/advection_auto_clone/const_indices.F90 b/end-to-end-tests/advection_auto_clone/const_indices.F90 new file mode 100644 index 00000000..5c77e29c --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/const_indices.F90 @@ -0,0 +1,95 @@ +! Test collection of constituent indices +! + +module const_indices + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: const_indices_init + public :: const_indices_run + +contains + + !> \section arg_table_const_indices_run Argument Table + !! \htmlinclude arg_table_const_indices_run.html + !! + subroutine const_indices_run(const_std_name, num_consts, test_stdname_array, & + const_index, const_inds, errmsg, errcode) + use ccpp_constituent_prop_mod, only: int_unassigned + use ccpp_scheme_utils, only: ccpp_constituent_index + use ccpp_scheme_utils, only: ccpp_constituent_indices + + character(len=*), intent(in) :: const_std_name + integer, intent(in) :: num_consts + character(len=*), intent(in) :: test_stdname_array(:) + integer, intent(out) :: const_index + integer, intent(out) :: const_inds(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errcode + !---------------------------------------------------------------- + + integer :: indx + integer :: test_indx + + errmsg = '' + errcode = 0 + + ! Find the constituent index for + call ccpp_constituent_index(const_std_name, const_index, errcode, errmsg) + if (errcode == 0) then + call ccpp_constituent_indices(test_stdname_array, const_inds, errcode, errmsg) + end if + ! Check that a non-registered constituent is detectable but + ! does not cause an error + if (errcode == 0) then + call ccpp_constituent_index('unobtainium', test_indx, errcode, errmsg) + if (test_indx /= int_unassigned) then + if (errcode == 0) then + ! Do not add an error if one is already reported + errcode = 2 + write(errmsg, '(2a,i0,a,i0)') "ccpp_constituent_index called for ", & + "'unobtainium' returned an index of ", test_indx, ", not ", & + int_unassigned + end if + end if + end if + + end subroutine const_indices_run + + !> \section arg_table_const_indices_init Argument Table + !! \htmlinclude arg_table_const_indices_init.html + !! + subroutine const_indices_init(const_std_name, num_consts, test_stdname_array, & + const_index, const_inds, errmsg, errcode) + use ccpp_scheme_utils, only: ccpp_constituent_index, & + ccpp_constituent_indices + + character(len=*), intent(in) :: const_std_name + integer, intent(in) :: num_consts + character(len=*), intent(in) :: test_stdname_array(:) + integer, intent(out) :: const_index + integer, intent(out) :: const_inds(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errcode + !---------------------------------------------------------------- + + integer :: indx + + errmsg = '' + errcode = 0 + + ! Find the constituent index for + call ccpp_constituent_index(const_std_name, const_index, errcode, errmsg) + if (errcode == 0) then + call ccpp_constituent_indices(test_stdname_array, const_inds, errcode, errmsg) + end if + + end subroutine const_indices_init + + !! @} + !! @} + +end module const_indices diff --git a/end-to-end-tests/advection_auto_clone/const_indices.meta b/end-to-end-tests/advection_auto_clone/const_indices.meta new file mode 100644 index 00000000..147e2ccb --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/const_indices.meta @@ -0,0 +1,108 @@ +# const_indices just returns some constituent indices as a test +[ccpp-table-properties] + name = const_indices + type = scheme +[ccpp-arg-table] + name = const_indices_run + type = scheme +[ const_std_name ] + standard_name = test_banana_name + type = character | kind = len=* + units = 1 + dimensions = () + protected = true + intent = in +[ num_consts ] + standard_name = banana_array_dim + long_name = Size of test_banana_name_array + units = 1 + dimensions = () + type = integer + intent = in +[ test_stdname_array ] + standard_name = test_banana_name_array + type = character | kind = len=* + units = count + dimensions = (banana_array_dim) + intent = in +[ const_index ] + standard_name = test_banana_constituent_index + long_name = Constituent index + units = 1 + dimensions = () + type = integer + intent = out +[ const_inds ] + standard_name = test_banana_constituent_indices + long_name = Array of constituent indices + units = 1 + dimensions = (banana_array_dim) + type = integer + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errcode ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = const_indices_init + type = scheme +[ const_std_name ] + standard_name = test_banana_name + type = character | kind = len=* + units = 1 + dimensions = () + protected = true + intent = in +[ num_consts ] + standard_name = banana_array_dim + long_name = Size of test_banana_name_array + units = 1 + dimensions = () + type = integer + intent = in +[ test_stdname_array ] + standard_name = test_banana_name_array + type = character | kind = len=* + units = count + dimensions = (banana_array_dim) + intent = in +[ const_index ] + standard_name = test_banana_constituent_index + long_name = Constituent index + units = 1 + dimensions = () + type = integer + intent = out +[ const_inds ] + standard_name = test_banana_constituent_indices + long_name = Array of constituent indices + units = 1 + dimensions = (banana_array_dim) + type = integer + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errcode ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/advection_auto_clone/dlc_liq.F90 b/end-to-end-tests/advection_auto_clone/dlc_liq.F90 new file mode 100644 index 00000000..134e3aed --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/dlc_liq.F90 @@ -0,0 +1,41 @@ +! Test parameterization with a runtime constituents +! properties object outside of the register phase + +module dlc_liq + + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + implicit none + private + + public :: dlc_liq_init + +contains + + !> \section arg_table_dlc_liq_init Argument Table + !! \htmlinclude arg_table_dlc_liq_init.html + !! + subroutine dlc_liq_init(dyn_const, errmsg, errcode) + type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errcode + + character(len=256) :: stdname + + errmsg = '' + errcode = 0 + allocate(dyn_const(1), stat=errcode) + if (errcode /= 0) then + errmsg = 'Error allocating dyn_const in dlc_liq_init' + return + end if + call dyn_const(1)%instantiate(std_name="dyn_const3", long_name='dyn const3', & + diag_name='DYNCONST3', units='kg kg-1', default_value=1._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + errcode=errcode, errmsg=errmsg) + call dyn_const(1)%standard_name(stdname, errcode=errcode, errmsg=errmsg) + + end subroutine dlc_liq_init + +end module dlc_liq diff --git a/test/unit_tests/sample_scheme_files/invalid_subr_stmnt.meta b/end-to-end-tests/advection_auto_clone/dlc_liq.meta similarity index 56% rename from test/unit_tests/sample_scheme_files/invalid_subr_stmnt.meta rename to end-to-end-tests/advection_auto_clone/dlc_liq.meta index 30bdc1e9..41a69db9 100644 --- a/test/unit_tests/sample_scheme_files/invalid_subr_stmnt.meta +++ b/end-to-end-tests/advection_auto_clone/dlc_liq.meta @@ -1,11 +1,17 @@ +# dlc_liq is a scheme that has a ccpp_constituent_properties_t variable +# outside of the register phase [ccpp-table-properties] - name = invalid_subr_stmnt + name = dlc_liq type = scheme - -######################################################################## [ccpp-arg-table] - name = invalid_subr_stmnt_init + name = dlc_liq_init type = scheme +[ dyn_const ] + standard_name = dynamic_constituents_for_dlc_liq + dimensions = (:) + type = ccpp_constituent_properties_t + intent = out + allocatable = true [ errmsg ] standard_name = ccpp_error_message long_name = Error message for error handling in CCPP @@ -14,7 +20,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 diff --git a/end-to-end-tests/advection_auto_clone/test_advection_host_integration.F90 b/end-to-end-tests/advection_auto_clone/test_advection_host_integration.F90 new file mode 100644 index 00000000..f1f73576 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/test_advection_host_integration.F90 @@ -0,0 +1,79 @@ +program test + use test_prog, only: test_host, & + suite_info, & + cm, & + cs + + implicit none + + character(len=cs), target :: test_parts1(1) + character(len=cm), target :: test_invars1(11) + character(len=cm), target :: test_outvars1(13) + character(len=cm), target :: test_reqvars1(18) + + type(suite_info) :: test_suites(1) + logical :: run_okay + + test_parts1 = (/ 'physics '/) + test_invars1 = (/ & + 'banana_array_dim ', & + 'cloud_ice_dry_mixing_ratio ', & + 'cloud_liquid_dry_mixing_ratio ', & + 'surface_air_pressure ', & + 'temperature ', & + 'time_step_for_physics ', & + 'water_temperature_at_freezing ', & + 'ccpp_constituent_tendencies ', & + 'ccpp_constituents ', & + 'number_of_ccpp_constituents ', & + 'water_vapor_specific_humidity ' /) + test_outvars1 = (/ & + 'ccpp_error_message ', & + 'ccpp_error_code ', & + 'temperature ', & + 'water_vapor_specific_humidity ', & + 'cloud_liquid_dry_mixing_ratio ', & + 'ccpp_constituent_tendencies ', & + 'ccpp_constituents ', & + 'dynamic_constituents_for_cld_liq ', & + 'dynamic_constituents_for_cld_ice ', & + 'tendency_of_cloud_liquid_dry_mixing_ratio', & + 'test_banana_constituent_index ', & + 'test_banana_constituent_indices ', & + 'cloud_ice_dry_mixing_ratio ' /) + test_reqvars1 = (/ & + 'banana_array_dim ', & + 'surface_air_pressure ', & + 'temperature ', & + 'time_step_for_physics ', & + 'cloud_liquid_dry_mixing_ratio ', & + 'tendency_of_cloud_liquid_dry_mixing_ratio', & + 'cloud_ice_dry_mixing_ratio ', & + 'dynamic_constituents_for_cld_liq ', & + 'dynamic_constituents_for_cld_ice ', & + 'water_temperature_at_freezing ', & + 'ccpp_constituent_tendencies ', & + 'ccpp_constituents ', & + 'number_of_ccpp_constituents ', & + 'test_banana_constituent_index ', & + 'test_banana_constituent_indices ', & + 'water_vapor_specific_humidity ', & + 'ccpp_error_message ', & + 'ccpp_error_code ' /) + + ! Setup expected test suite info + test_suites(1)%suite_name = 'cld_suite' + test_suites(1)%suite_parts => test_parts1 + test_suites(1)%suite_input_vars => test_invars1 + test_suites(1)%suite_output_vars => test_outvars1 + test_suites(1)%suite_required_vars => test_reqvars1 + + call test_host(run_okay, test_suites) + + if (run_okay) then + stop 0 + else + stop -1 + end if + +end program test diff --git a/test/advection_test/test_host.F90 b/end-to-end-tests/advection_auto_clone/test_host.F90 similarity index 56% rename from test/advection_test/test_host.F90 rename to end-to-end-tests/advection_auto_clone/test_host.F90 index cc8bbf89..7845ca79 100644 --- a/test/advection_test/test_host.F90 +++ b/end-to-end-tests/advection_auto_clone/test_host.F90 @@ -27,26 +27,26 @@ module test_prog private :: check_suite private :: advect_constituents ! Move data around - private :: check_errflg + private :: check_errcode contains - subroutine check_errflg(subname, errflg, errmsg, errflg_final) - ! If errflg is not zero, print an error message + subroutine check_errcode(subname, errcode, errmsg, errcode_final) + ! If errcode is not zero, print an error message character(len=*), intent(in) :: subname - integer, intent(in) :: errflg + integer, intent(in) :: errcode character(len=*), intent(in) :: errmsg - integer, intent(out) :: errflg_final + integer, intent(out) :: errcode_final - if (errflg /= 0) then - write(6, '(a,i0,4a)') "Error ", errflg, " from ", trim(subname), & + if (errcode /= 0) then + write(6, '(a,i0,4a)') "Error ", errcode, " from ", trim(subname), & ':', trim(errmsg) !Notify test script that a failure occurred: - errflg_final = -1 !Notify test script that a failure occured + errcode_final = -1 !Notify test script that a failure occured end if - end subroutine check_errflg + end subroutine check_errcode logical function check_suite(test_suite) use test_host_ccpp_cap, only: ccpp_physics_suite_part_list @@ -57,20 +57,20 @@ logical function check_suite(test_suite) type(suite_info), intent(in) :: test_suite ! Local variables logical :: check - integer :: errflg + integer :: errcode character(len=512) :: errmsg character(len=128), allocatable :: test_list(:) check_suite = .true. ! First, check the suite parts call ccpp_physics_suite_part_list(test_suite%suite_name, test_list, & - errmsg, errflg) - if (errflg == 0) then + errmsg, errcode) + if (errcode == 0) then check = check_list(test_list, test_suite%suite_parts, 'part names', & suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR ', errcode, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then @@ -78,13 +78,13 @@ logical function check_suite(test_suite) end if ! Check the input variables call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg, input_vars=.true., output_vars=.false.) - if (errflg == 0) then + errmsg, errcode, input_vars=.true., output_vars=.false.) + if (errcode == 0) then check = check_list(test_list, test_suite%suite_input_vars, & 'input variable names', suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR ', errcode, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then @@ -92,13 +92,13 @@ logical function check_suite(test_suite) end if ! Check the output variables call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg, input_vars=.false., output_vars=.true.) - if (errflg == 0) then + errmsg, errcode, input_vars=.false., output_vars=.true.) + if (errcode == 0) then check = check_list(test_list, test_suite%suite_output_vars, & 'output variable names', suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR ', errcode, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then @@ -106,13 +106,13 @@ logical function check_suite(test_suite) end if ! Check all required variables call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg) - if (errflg == 0) then + errmsg, errcode) + if (errcode == 0) then check = check_list(test_list, test_suite%suite_required_vars, & 'required variable names', suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR ', errcode, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then @@ -148,21 +148,23 @@ subroutine test_host(retval, test_suites) std_name_array, & const_std_name use test_host_data, only: check_constituent_indices - use test_host_ccpp_cap, only: test_host_ccpp_deallocate_dynamic_constituents - use test_host_ccpp_cap, only: test_host_ccpp_register_constituents - use test_host_ccpp_cap, only: test_host_ccpp_is_scheme_constituent - use test_host_ccpp_cap, only: test_host_ccpp_initialize_constituents - use test_host_ccpp_cap, only: test_host_ccpp_number_constituents - use test_host_ccpp_cap, only: test_host_constituents_array - use test_host_ccpp_cap, only: test_host_ccpp_physics_register - use test_host_ccpp_cap, only: test_host_ccpp_physics_initialize - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_initial - use test_host_ccpp_cap, only: test_host_ccpp_physics_run - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_final - use test_host_ccpp_cap, only: test_host_ccpp_physics_finalize + use test_host_ccpp_cap, only: ccpp_deallocate_dynamic_constituents + use test_host_ccpp_cap, only: ccpp_register_constituents + use test_host_ccpp_cap, only: ccpp_is_scheme_constituent + use test_host_ccpp_cap, only: ccpp_initialize_constituents + use test_host_ccpp_cap, only: ccpp_number_constituents + use test_host_ccpp_cap, only: ccpp_constituents_array + use test_host_ccpp_cap, only: ccpp_register + use test_host_ccpp_cap, only: ccpp_init + use test_host_ccpp_cap, only: ccpp_physics_init + use test_host_ccpp_cap, only: ccpp_physics_timestep_init + use test_host_ccpp_cap, only: ccpp_physics_run + use test_host_ccpp_cap, only: ccpp_physics_timestep_final + use test_host_ccpp_cap, only: ccpp_physics_final + use test_host_ccpp_cap, only: ccpp_final use test_host_ccpp_cap, only: ccpp_physics_suite_list - use test_host_ccpp_cap, only: test_host_const_get_index - use test_host_ccpp_cap, only: test_host_model_const_properties + use test_host_ccpp_cap, only: ccpp_const_get_index + use test_host_ccpp_cap, only: ccpp_model_const_properties use test_utils, only: check_list type(suite_info), intent(in) :: test_suites(:) @@ -185,8 +187,8 @@ subroutine test_host(retval, test_suites) character(len=256) :: const_str character(len=512) :: errmsg character(len=512) :: expected_error - integer :: errflg - integer :: errflg_final ! Used to notify testing script of test failure + integer :: errcode + integer :: errcode_final ! Used to notify testing script of test failure real(kind=kind_phys), pointer :: const_ptr(:, :, :) real(kind=kind_phys) :: default_value real(kind=kind_phys) :: check_value @@ -195,7 +197,7 @@ subroutine test_host(retval, test_suites) ! Initialized "final" error flag used to report a failure to the larged ! testing script: - errflg_final = 0 + errcode_final = 0 ! Gather and test the inspection routines num_suites = size(test_suites) @@ -224,36 +226,36 @@ subroutine test_host(retval, test_suites) return end if - errflg = 0 + errcode = 0 errmsg = '' ! Check that is_scheme_constituent works as expected - call test_host_ccpp_is_scheme_constituent('specific_humidity', & - is_constituent, errflg, errmsg) - call check_errflg(subname // "_ccpp_is_scheme_constituent", errflg, & - errmsg, errflg_final) + call ccpp_is_scheme_constituent('specific_humidity', & + is_constituent, errcode, errmsg) + call check_errcode(subname // "_ccpp_is_scheme_constituent", errcode, & + errmsg, errcode_final) ! specific_humidity should not be an existing constituent if (is_constituent) then write(6, *) "ERROR: specific humidity is already a constituent" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - call test_host_ccpp_is_scheme_constituent('cloud_ice_dry_mixing_ratio', & - is_constituent, errflg, errmsg) - call check_errflg(subname // "_ccpp_is_scheme_constituent", errflg, & - errmsg, errflg_final) + call ccpp_is_scheme_constituent('cloud_ice_dry_mixing_ratio', & + is_constituent, errcode, errmsg) + call check_errcode(subname // "_ccpp_is_scheme_constituent", errcode, & + errmsg, errcode_final) ! cloud_ice_dry_mixing_ratio should be an existing constituent if (.not. is_constituent) then write(6, *) "ERROR: cloud_ice_dry_mixing ratio not found in ", & "host cap constituent list" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if ! Use the suite information to call the register phase do sind = 1, num_suites - if (errflg == 0) then - call test_host_ccpp_physics_register( & - test_suites(sind)%suite_name, errmsg, errflg) - if (errflg /= 0) then + if (errcode == 0) then + call ccpp_register(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(4a)') 'ERROR in register of ', & trim(test_suites(sind)%suite_name), ': ', trim(errmsg) exit @@ -274,19 +276,19 @@ subroutine test_host(retval, test_suites) long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & vertical_dim="vertical_layer_dimension", advected=.true., & min_value=1000._kind_phys, molar_mass=2000._kind_phys, & - errcode=errflg, errmsg=errmsg) + errcode=errcode, errmsg=errmsg) call host_constituents(2)%instantiate(std_name="specific_humidity", & long_name="Specific humidity", diag_name='H2O', units="kg kg", & vertical_dim="vertical_layer_dimension", advected=.true., & min_value=1000._kind_phys, molar_mass=2000._kind_phys, & - errcode=errflg, errmsg=errmsg) - call check_errflg(subname // '.initialize', errflg, errmsg, errflg_final) - if (errflg == 0) then - call test_host_ccpp_register_constituents(host_constituents, & - errmsg=errmsg, errflg=errflg) + errcode=errcode, errmsg=errmsg) + call check_errcode(subname // '.initialize', errcode, errmsg, errcode_final) + if (errcode == 0) then + call ccpp_register_constituents(host_constituents, & + errmsg=errmsg, errcode=errcode) end if ! Check the error - if (errflg == 0) then + if (errcode == 0) then write(6, '(2a)') 'ERROR register_constituents: expected this error: ', & trim(expected_error) else @@ -295,17 +297,18 @@ subroutine test_host(retval, test_suites) trim(expected_error), ' Got: ', trim(errmsg) end if end if + ! Now try again but with a compatible constituent - should be ignored when ! the constituents object is created ! Use the suite information to call the register phase - errflg = 0 - call test_host_ccpp_deallocate_dynamic_constituents() + errcode = 0 + call ccpp_deallocate_dynamic_constituents() deallocate(host_constituents) do sind = 1, num_suites - if (errflg == 0) then - call test_host_ccpp_physics_register( & - test_suites(sind)%suite_name, errmsg, errflg) - if (errflg /= 0) then + if (errcode == 0) then + call ccpp_register(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(4a)') 'ERROR in register of ', & trim(test_suites(sind)%suite_name), ': ', trim(errmsg) exit @@ -317,27 +320,27 @@ subroutine test_host(retval, test_suites) long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & vertical_dim="vertical_layer_dimension", advected=.true., & min_value=1000._kind_phys, molar_mass=2000._kind_phys, & - errcode=errflg, errmsg=errmsg) + errcode=errcode, errmsg=errmsg) call host_constituents(2)%instantiate(std_name="specific_humidity", & long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & vertical_dim="vertical_layer_dimension", advected=.true., & min_value=1000._kind_phys, molar_mass=2000._kind_phys, & - errcode=errflg, errmsg=errmsg) - call check_errflg(subname // '.initialize', errflg, errmsg, errflg_final) - if (errflg == 0) then - call test_host_ccpp_register_constituents(host_constituents, & - errmsg=errmsg, errflg=errflg) + errcode=errcode, errmsg=errmsg) + call check_errcode(subname // '.initialize', errcode, errmsg, errcode_final) + if (errcode == 0) then + call ccpp_register_constituents(host_constituents, & + errmsg=errmsg, errcode=errcode) end if - if (errflg /= 0) then + if (errcode /= 0) then write(6, '(2a)') 'ERROR register_constituents: ', trim(errmsg) retval = .false. return end if ! Check number of advected constituents - if (errflg == 0) then - call test_host_ccpp_number_constituents(num_advected, errmsg=errmsg, & - errflg=errflg) - call check_errflg(subname // ".num_advected", errflg, errmsg, errflg_final) + if (errcode == 0) then + call ccpp_number_constituents(num_advected, errmsg=errmsg, & + errcode=errcode) + call check_errcode(subname // ".num_advected", errcode, errmsg, errcode_final) end if if (num_advected /= 6) then write(6, '(a,i0)') "ERROR: num advected constituents = ", num_advected @@ -345,61 +348,60 @@ subroutine test_host(retval, test_suites) return end if ! Initialize constituent data - call test_host_ccpp_initialize_constituents(ncols, pver, errflg, errmsg) + call ccpp_initialize_constituents(ncols=ncols, num_layers=pver, errcode=errcode, errmsg=errmsg) ! Stop tests here if initialization failed (as all other tests will likely ! fail as well: - if (errflg /= 0) then + if (errcode /= 0) then retval = .false. return end if ! Initialize our 'data' - const_ptr => test_host_constituents_array() + const_ptr => ccpp_constituents_array() ! Check if the specific humidity index can be found: - call test_host_const_get_index('specific_humidity', index, & - errflg, errmsg) - call check_errflg(subname // ".index_specific_humidity", errflg, errmsg, & - errflg_final) + call ccpp_const_get_index('specific_humidity', const_index=index, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_specific_humidity", errcode, errmsg, & + errcode_final) ! Check if the cloud liquid index can be found: - call test_host_const_get_index('cloud_liquid_dry_mixing_ratio', & - index_liq, errflg, errmsg) - call check_errflg(subname // ".index_cld_liq", errflg, errmsg, & - errflg_final) + call ccpp_const_get_index(stdname='cloud_liquid_dry_mixing_ratio', & + const_index=index_liq, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_cld_liq", errcode, errmsg, & + errcode_final) ! Check if the cloud ice index can be found: - call test_host_const_get_index('cloud_ice_dry_mixing_ratio', & - index_ice, errflg, errmsg) - call check_errflg(subname // ".index_cld_ice", errflg, errmsg, & - errflg_final) + call ccpp_const_get_index(stdname='cloud_ice_dry_mixing_ratio', & + const_index=index_ice, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_cld_ice", errcode, errmsg, & + errcode_final) ! Check if the dynamic constituents indices can be found - call test_host_const_get_index('dyn_const1', index_dyn1, errflg, errmsg) - call check_errflg(subname // ".index_dyn_const1", errflg, errmsg, & - errflg_final) - call test_host_const_get_index('dyn_const2_wrt_moist_air', index_dyn2, errflg, errmsg) - call check_errflg(subname // ".index_dyn_const2", errflg, errmsg, & - errflg_final) - call test_host_const_get_index('dyn_const3_wrt_moist_air_and_condensed_water', index_dyn3, errflg, errmsg) - call check_errflg(subname // ".index_dyn_const3", errflg, errmsg, & - errflg_final) + call ccpp_const_get_index(stdname='dyn_const1', const_index=index_dyn1, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_dyn_const1", errcode, errmsg, & + errcode_final) + call ccpp_const_get_index(stdname='dyn_const2_wrt_moist_air', const_index=index_dyn2, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_dyn_const2", errcode, errmsg, & + errcode_final) + call ccpp_const_get_index(stdname='dyn_const3_wrt_moist_air_and_condensed_water', const_index=index_dyn3, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_dyn_const3", errcode, errmsg, & + errcode_final) ! Load up the test array indices - call test_host_const_get_index(const_std_name, test_scalar_const_index, errflg, errmsg) - call check_errflg(subname // "." // const_std_name, errflg, errmsg, & - errflg_final) + call ccpp_const_get_index(stdname=const_std_name, const_index=test_scalar_const_index, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // "." // const_std_name, errcode, errmsg, & + errcode_final) do sind = 1, num_consts - call test_host_const_get_index(std_name_array(sind), & - test_const_indices(sind), errflg, errmsg) - call check_errflg(subname // "." // std_name_array(sind), errflg, errmsg, & - errflg_final) + call ccpp_const_get_index(stdname=std_name_array(sind), & + const_index=test_const_indices(sind), errcode=errcode, errmsg=errmsg) + call check_errcode(subname // "." // std_name_array(sind), errcode, errmsg, & + errcode_final) end do ! Stop tests here if the index checks failed, as all other tests will ! likely fail as well: - if (errflg_final /= 0) then + if (errcode_final /= 0) then retval = .false. return end if @@ -409,268 +411,268 @@ subroutine test_host(retval, test_suites) ! Check some constituent properties ! ++++++++++++++++++++++++++++++++++ - const_props => test_host_model_const_properties() + const_props => ccpp_model_const_properties() ! Standard name: - call const_props(index)%standard_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index)%standard_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get standard_name for specific_humidity, index = ", & index, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'specific_humidity') then write(6, *) "ERROR: standard name, '", trim(const_str), & "' should be 'specific_humidity'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check standard name for a dynamic constituent - call const_props(index_dyn2)%standard_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn2)%standard_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get standard_name for dyn_const2, index = ", & index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'dyn_const2_wrt_moist_air') then write(6, *) "ERROR: standard name, '", trim(const_str), & "' should be 'dyn_const2_wrt_moist_air'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Long name: - call const_props(index_liq)%long_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_liq)%long_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get long_name for cld_liq index = ", & index_liq, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'Cloud liquid dry mixing ratio') then write(6, *) "ERROR: long name, '", trim(const_str), & "' should be 'Cloud liquid dry mixing ratio'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check long name for a dynamic constituent - call const_props(index_dyn1)%long_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn1)%long_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get long_name for dyn_const1 index = ", & index_dyn1, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'dyn const1') then write(6, *) "ERROR: long name, '", trim(const_str), & "' should be 'dyn const1'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Diagnostic name: - call const_props(index_liq)%diagnostic_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_liq)%diagnostic_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get diagnostic name for cld_liq index = ", & index_liq, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'CLDLIQ') then write(6, *) "ERROR: diagnostic name, '", trim(const_str), & "' should be 'CLDLIQ'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check default diagnostic name is set correctly - call const_props(index_ice)%diagnostic_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_ice)%diagnostic_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get diagnostic name for cld_ice index = ", & index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'cld_ice_array') then write(6, *) "ERROR: diagnostic name, '", trim(const_str), & "' should be 'cld_ice_array'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check diagnostic name of a dynamic constituent - call const_props(index_dyn2)%diagnostic_name(const_str, errflg, & + call const_props(index_dyn2)%diagnostic_name(const_str, errcode, & errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get diagnostic name for dyn_const2 index = ", & index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'DYNCONST2') then write(6, *) "ERROR: diagnostic name, '", trim(const_str), & "' should be 'DYNCONST2'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Mass mixing ratio: - call const_props(index_ice)%is_mass_mixing_ratio(const_log, errflg, & + call const_props(index_ice)%is_mass_mixing_ratio(const_log, errcode, & errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get mass mixing ratio prop for cld_ice index = ", & index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (.not. const_log) then write(6, *) "ERROR: cloud ice is not a mass mixing_ratio" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check mass mixing ratio for a dynamic constituent - call const_props(index_dyn2)%is_mass_mixing_ratio(const_log, errflg, & + call const_props(index_dyn2)%is_mass_mixing_ratio(const_log, errcode, & errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get mass mixing ratio prop for dyn_const2 index = ", & index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (.not. const_log) then write(6, *) "ERROR: dyn_const2 is not a mass mixing_ratio" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Dry mixing ratio: - call const_props(index_ice)%is_dry(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_ice)%is_dry(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get dry prop for cld_ice index = ", index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (.not. const_log) then write(6, *) "ERROR: cloud ice mass_mixing_ratio is not dry" - errflg_final = -1 + errcode_final = -1 end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check wet mixing ratio for dynamic constituent 1 - call const_props(index_dyn1)%is_dry(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn1)%is_dry(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get dry prop for dyn_const1 index = ", index_dyn1, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (const_log) then write(6, *) "ERROR: dyn_const1 is dry and should be wet" - errflg_final = -1 + errcode_final = -1 end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if - call const_props(index_dyn1)%is_wet(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn1)%is_wet(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get wet prop for dyn_const1 index = ", index_dyn1, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (.not. const_log) then write(6, *) "ERROR: dyn_const1 is not wet but should be" - errflg_final = -1 + errcode_final = -1 end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check moist mixing ratio for dynamic constituent 2 - call const_props(index_dyn2)%is_dry(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn2)%is_dry(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get dry prop for dyn_const2 index = ", index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (const_log) then write(6, *) "ERROR: dyn_const2 is dry and should be moist" - errflg_final = -1 + errcode_final = -1 end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if - call const_props(index_dyn2)%is_moist(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn2)%is_moist(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get moist prop for dyn_const2 index = ", index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (.not. const_log) then write(6, *) "ERROR: dyn_const2 is not moist but should be" - errflg_final = -1 + errcode_final = -1 end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check dry mixing ratio for dynamic constituent 3 - call const_props(index_dyn3)%is_dry(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn3)%is_dry(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get dry prop for dyn_const3 index = ", index_dyn3, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (.not. const_log) then write(6, *) "ERROR: dyn_const3 is not dry and should be" - errflg_final = -1 + errcode_final = -1 end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! ------------------- @@ -680,71 +682,71 @@ subroutine test_host(retval, test_suites) ! ------------------- ! Check that a constituent's minimum value defaults to zero: - call const_props(index_dyn2)%minimum(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn2)%minimum(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get minimum value for dyn_const2 index = ", index_dyn2, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (check_value /= 0._kind_phys) then ! Should be zero write(6, *) "ERROR: 'minimum' should default to zero for all ", & "constituents unless set by host model or scheme metadata." - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check that a constituent instantiated with a specified minimum value ! actually contains that minimum value property: - call const_props(index_dyn1)%minimum(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn1)%minimum(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get minimum value for dyn_const1 index = ", index_dyn1, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (check_value /= 1000._kind_phys) then !Should be 1000 write(6, *) "ERROR: 'minimum' should give a value of 1000 ", & "for dyn_const1, as was set during instantiation." - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check that setting a constituent's minimum value works ! as expected: - call const_props(index_dyn1)%set_minimum(1._kind_phys, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn1)%set_minimum(1._kind_phys, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to set minimum value for dyn_const1 index = ", index_dyn1, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then - call const_props(index_dyn1)%minimum(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + if (errcode == 0) then + call const_props(index_dyn1)%minimum(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & " trying to get minimum value for dyn_const1 index = ", & index_dyn1, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if end if - if (errflg == 0) then + if (errcode == 0) then if (check_value /= 1._kind_phys) then ! Should now be one write(6, *) "ERROR: 'set_minimum' did not set constituent", & " minimum value correctly." - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! ---------------------- @@ -753,52 +755,52 @@ subroutine test_host(retval, test_suites) ! Check that a constituent instantiated with a specified molecular ! weight actually contains that molecular weight property value: - call const_props(index)%molar_mass(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index)%molar_mass(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get molecular weight for specific humidity index = ", & index, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (check_value /= 2000._kind_phys) then ! Should be 2000 write(6, *) "ERROR: 'molar_mass' should give a value of 2000 ", & "for specific humidity, as was set during instantiation." - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check that setting a constituent's molecular weight works ! as expected: - call const_props(index_ice)%set_molar_mass(1._kind_phys, errflg, & + call const_props(index_ice)%set_molar_mass(1._kind_phys, errcode, & errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to set molecular weight for cld_ice index = ", index_ice, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then - call const_props(index_ice)%molar_mass(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + if (errcode == 0) then + call const_props(index_ice)%molar_mass(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & " trying to get molecular weight for cld_ice index = ", & index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if end if - if (errflg == 0) then + if (errcode == 0) then if (check_value /= 1._kind_phys) then ! Should be equal to one write(6, *) "ERROR: 'set_molar_mass' did not set constituent", & " molecular weight value correctly." - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! ------------------- @@ -806,51 +808,51 @@ subroutine test_host(retval, test_suites) ! ------------------- ! Check that being thermodynamically active defaults to False: - call const_props(index_ice)%is_thermo_active(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_ice)%is_thermo_active(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get thermo_active prop for cld_ice index = ", index_ice, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (check) then ! Should be False write(6, *) "ERROR: 'is_thermo_active' should default to False ", & "for all constituents unless set by host model." - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check that setting a constituent to be thermodynamically active works ! as expected: - call const_props(index_ice)%set_thermo_active(.true., errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_ice)%set_thermo_active(.true., errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to set thermo_active prop for cld_ice index = ", index_ice, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then - call const_props(index_ice)%is_thermo_active(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + if (errcode == 0) then + call const_props(index_ice)%is_thermo_active(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & " trying to get thermo_active prop for cld_ice index = ", & index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if end if - if (errflg == 0) then + if (errcode == 0) then if (.not. check) then ! Should now be True write(6, *) "ERROR: 'set_thermo_active' did not set", & " thermo_active constituent property correctly." - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! ------------------- @@ -859,149 +861,165 @@ subroutine test_host(retval, test_suites) ! ------------------- ! Check that being a water species defaults to False: - call const_props(index_liq)%is_water_species(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_liq)%is_water_species(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get water_species prop for cld_liq index = ", index_liq, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (check) then ! Should be False write(6, *) "ERROR: 'is_water_species' should default to False ", & "for all constituents unless set by host model." - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check that setting a constituent to be a water species works ! as expected: - call const_props(index_liq)%set_water_species(.true., errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_liq)%set_water_species(.true., errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to set water_species prop for cld_liq index = ", index_liq, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then - call const_props(index_liq)%is_water_species(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + if (errcode == 0) then + call const_props(index_liq)%is_water_species(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & " trying to get water_species prop for cld_liq index = ", & index_liq, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if end if - if (errflg == 0) then + if (errcode == 0) then if (.not. check) then ! Should now be True write(6, *) "ERROR: 'set_water_species' did not set", & " water_species constituent property correctly." - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check that setting a constituent to be a water species via the ! instantiate call works as expected - call const_props(index_dyn1)%is_water_species(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + call const_props(index_dyn1)%is_water_species(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & "trying to get water_species prop for dyn_const1 index = ", & index_dyn1, trim(errmsg) end if - if (errflg == 0) then + if (errcode == 0) then if (.not. check) then ! Should now be True write(6, *) "ERROR: 'water_species=.true. did not set", & " water_species constituent property correctly" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if - call const_props(index_dyn2)%is_water_species(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + call const_props(index_dyn2)%is_water_species(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & "trying to get water_species prop for dyn_const2 index = ", & index_dyn2, trim(errmsg) end if - if (errflg == 0) then + if (errcode == 0) then if (check) then ! Should now be False write(6, *) "ERROR: 'water_species=.false. did not set", & " water_species constituent property correctly" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! ------------------- ! Check that setting a constituent's default value works as expected - call const_props(index_liq)%has_default(has_default, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_liq)%has_default(has_default, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to check for default for cld_liq index = ", index_liq, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (has_default) then write(6, *) "ERROR: cloud liquid mass_mixing_ratio should not have default but does" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if - call const_props(index_ice)%has_default(has_default, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_ice)%has_default(has_default, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to check for default for cld_ice index = ", index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (.not. has_default) then - write(6, *) "ERROR: cloud ice mass_mixing_ratio should have default but doesn't" - errflg_final = -1 ! Notify test script that a failure occurred + write(6, *) "ERROR: cloud ice_dry_mixing_ratio should have default but doesn't" + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if - call const_props(index_ice)%default_value(default_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_ice)%default_value(default_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to grab default for cld_ice index = ", index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (default_value /= 0.0_kind_phys) then write(6, *) "ERROR: cloud ice mass_mixing_ratio default is ", default_value, & " but should be 0.0" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! ++++++++++++++++++++++++++++++++++ ! Set error flag to the "final" value, because any error ! above will likely result in a large number of failures ! below: - errflg = errflg_final + errcode = errcode_final + + ! Call ccpp_init + do sind = 1, num_suites + if (errcode == 0) then + call ccpp_init(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(6, '(4a)') 'ERROR in initialize of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + exit + end if + end if + end do - ! Use the suite information to setup the run + ! Call ccpp_physics_init do sind = 1, num_suites - if (errflg == 0) then - call test_host_ccpp_physics_initialize( & - test_suites(sind)%suite_name, errmsg, errflg) - if (errflg /= 0) then + if (errcode == 0) then + call ccpp_physics_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(4a)') 'ERROR in initialize of ', & trim(test_suites(sind)%suite_name), ': ', trim(errmsg) exit @@ -1011,18 +1029,21 @@ subroutine test_host(retval, test_suites) ! Check indices call check_constituent_indices(test_scalar_const_index, test_const_indices, & - errmsg, errflg) - call check_errflg(subname // " check suite indices", errflg, errmsg, & - errflg_final) + errmsg, errcode) + call check_errcode(subname // " check suite indices", errcode, errmsg, & + errcode_final) ! Loop over time steps do time_step = 1, num_time_steps ! Initialize the timestep do sind = 1, num_suites - if (errflg == 0) then - call test_host_ccpp_physics_timestep_initial( & - test_suites(sind)%suite_name, errmsg, errflg) - if (errflg /= 0) then + if (errcode == 0) then + call ccpp_physics_timestep_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & trim(errmsg) end if @@ -1030,19 +1051,21 @@ subroutine test_host(retval, test_suites) end do do col_start = 1, ncols, 5 - if (errflg /= 0) then + if (errcode /= 0) then continue end if col_end = min(col_start + 4, ncols) do sind = 1, num_suites do index = 1, size(test_suites(sind)%suite_parts) - if (errflg == 0) then - call test_host_ccpp_physics_run( & - test_suites(sind)%suite_name, & - test_suites(sind)%suite_parts(index), & - col_start, col_end, errmsg, errflg) - if (errflg /= 0) then + if (errcode == 0) then + call ccpp_physics_run( & + suite_name=test_suites(sind)%suite_name, & + group_name=test_suites(sind)%suite_parts(index), & + col_start=col_start, col_end=col_end, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(5a)') trim(test_suites(sind)%suite_name), & '/', trim(test_suites(sind)%suite_parts(index)),& ': ', trim(errmsg) @@ -1054,16 +1077,19 @@ subroutine test_host(retval, test_suites) end do ! Check indices call check_constituent_indices(test_scalar_const_index, test_const_indices, & - errmsg, errflg) - call check_errflg(subname // " check suite indices", errflg, errmsg, & - errflg_final) + errmsg, errcode) + call check_errcode(subname // " check suite indices", errcode, errmsg, & + errcode_final) do sind = 1, num_suites - if (errflg == 0) then - call test_host_ccpp_physics_timestep_final( & - test_suites(sind)%suite_name, errmsg, errflg) + if (errcode == 0) then + call ccpp_physics_timestep_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) end if - if (errflg /= 0) then + if (errcode /= 0) then write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & trim(errmsg) exit @@ -1071,43 +1097,63 @@ subroutine test_host(retval, test_suites) end do ! Run "dycore" - if (errflg == 0) then + if (errcode == 0) then call advect_constituents() end if end do ! End time step loop do sind = 1, num_suites - if (errflg == 0) then - call test_host_ccpp_physics_finalize( & - test_suites(sind)%suite_name, errmsg, errflg) - if (errflg /= 0) then + if (errcode == 0) then + call ccpp_physics_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_physics_final, ', & + 'Exiting...' + exit + end if + end if + end do + + do sind = 1, num_suites + if (errcode == 0) then + call ccpp_final(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & trim(errmsg) - write(6, '(2a)') 'An error occurred in ccpp_timestep_final, ', & + write(6, '(2a)') 'An error occurred in ccpp_final, ', & 'Exiting...' exit end if end if end do - if (errflg == 0) then + call ccpp_deallocate_dynamic_constituents() + deallocate(host_constituents) + + if (errcode == 0) then ! Run finished without error, check answers if (compare_data(num_advected)) then write(6, *) 'Answers are correct!' - errflg = 0 + errcode = 0 else write(6, *) 'Answers are not correct!' - errflg = -1 + errcode = -1 end if end if - ! Make sure "final" flag is non-zero if "errflg" is: - if (errflg /= 0) then - errflg_final = -1 ! Notify test script that a failure occured + ! Make sure "final" flag is non-zero if "errcode" is: + if (errcode /= 0) then + errcode_final = -1 ! Notify test script that a failure occured end if ! Set return value to False if any errors were found: - retval = errflg_final == 0 + retval = errcode_final == 0 end subroutine test_host diff --git a/end-to-end-tests/advection_auto_clone/test_host.meta b/end-to-end-tests/advection_auto_clone/test_host.meta new file mode 100644 index 00000000..e69dafd9 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/test_host.meta @@ -0,0 +1,70 @@ +[ccpp-table-properties] + name = suite_info + type = ddt +[ccpp-arg-table] + name = suite_info + type = ddt + +[ccpp-table-properties] + name = test_host + type = control +[ccpp-arg-table] + name = test_host + type = control +[ col_start ] + standard_name = horizontal_loop_begin + type = integer + units = count + dimensions = () + protected = True +[ col_end ] + standard_name = horizontal_loop_end + type = integer + units = count + dimensions = () + protected = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 +[ errcode ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer diff --git a/end-to-end-tests/advection_auto_clone/test_host_data.F90 b/end-to-end-tests/advection_auto_clone/test_host_data.F90 new file mode 100644 index 00000000..4bcb753b --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/test_host_data.F90 @@ -0,0 +1,96 @@ +module test_host_data + + use ccpp_kinds, only: kind_phys + + implicit none + + !> \section arg_table_physics_state Argument Table + !! \htmlinclude arg_table_physics_state.html + type physics_state + real(kind=kind_phys), allocatable :: ps(:) ! surface pressure + real(kind=kind_phys), allocatable :: temp(:, :) ! temperature + real(kind=kind_phys), dimension(:, :, :), pointer :: q => null() ! constituent array + end type physics_state + + !> \section arg_table_test_host_data Argument Table + !! \htmlinclude arg_table_test_host_data.html + integer, public, parameter :: num_consts = 3 + character(len=32), public, parameter :: std_name_array(num_consts) = (/ & + 'specific_humidity ', & + 'cloud_liquid_dry_mixing_ratio', & + 'cloud_ice_dry_mixing_ratio ' /) + character(len=32), public, parameter :: const_std_name = std_name_array(1) + + integer :: const_inds(num_consts) = -1 ! test array access from suite + integer :: const_index = -1 ! test scalar access from suite + + public :: allocate_physics_state + public :: check_constituent_indices + +contains + + subroutine check_constituent_indices(test_index, test_indices, errmsg, errcode) + ! Check constituent indices against what was found by suite + ! indices are passed in rather than looked up to avoid a dependency loop + ! Dummy arguments + integer, intent(in) :: test_index ! scalar const index from host + integer, intent(in) :: test_indices(:) ! array_test_indices from host + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errcode + + ! Local variable + integer :: indx + integer :: emstrt + + errcode = 0 + errmsg = '' + if (test_index /= const_index) then + emstrt = len_trim(errmsg) + 1 + write(errmsg(emstrt:), '(2a,i0,a,i0)') 'const_index_check for ', & + const_std_name, test_index, ' /= ', const_index + errcode = errcode + 1 + end if + do indx = 1, num_consts + if (test_indices(indx) /= const_inds(indx)) then + emstrt = len_trim(errmsg) + 1 + if (len_trim(errmsg) > 0) then + write(errmsg(emstrt:), '(", ")') + emstrt = emstrt + 2 + end if + write(errmsg(emstrt:), '(2a,i0,a,i0)') 'const_indices_check for ', & + std_name_array(indx), test_indices(indx), ' /= ', const_inds(indx) + errcode = errcode + 1 + end if + end do + + ! Reset for next test + const_index = -1 + const_inds = -1 + + end subroutine check_constituent_indices + + subroutine allocate_physics_state(cols, levels, constituents, state) + integer, intent(in) :: cols + integer, intent(in) :: levels + real(kind=kind_phys), pointer :: constituents(:, :, :) + type(physics_state), intent(out) :: state + + if (allocated(state%ps)) then + deallocate(state%ps) + end if + allocate(state%ps(cols)) + state%ps = 0.0_kind_phys + if (allocated(state%temp)) then + deallocate(state%temp) + end if + allocate(state%temp(cols, levels)) + if (associated(state%q)) then + ! Do not deallocate (we do not own this array) + nullify(state%q) + end if + ! Point to the advected constituents array + state%q => constituents + + end subroutine allocate_physics_state + +end module test_host_data diff --git a/end-to-end-tests/advection_auto_clone/test_host_data.meta b/end-to-end-tests/advection_auto_clone/test_host_data.meta new file mode 100644 index 00000000..960ce33e --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/test_host_data.meta @@ -0,0 +1,67 @@ +[ccpp-table-properties] + name = physics_state + type = ddt +[ccpp-arg-table] + name = physics_state + type = ddt +[ ps ] + standard_name = surface_air_pressure + type = real + kind = kind_phys + units = Pa + dimensions = (horizontal_dimension) +[ Temp ] + standard_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys +[ q ] + standard_name = constituent_mixing_ratio + type = real + kind = kind_phys + units = kg kg-1 moist or dry air depending on type + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_tracers) +[ q(:,:,index_of_water_vapor_specific_humidity) ] + standard_name = water_vapor_specific_humidity + type = real + kind = kind_phys + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + +[ccpp-table-properties] + name = test_host_data + type = host +[ccpp-arg-table] + name = test_host_data + type = host +[ num_consts ] + standard_name = banana_array_dim + long_name = Size of test_banana_name_array + units = 1 + dimensions = () + type = integer +[ std_name_array ] + standard_name = test_banana_name_array + type = character | kind = len=32 + units = count + dimensions = (banana_array_dim) + protected = true +[ const_std_name ] + standard_name = test_banana_name + type = character | kind = len=32 + units = 1 + dimensions = () + protected = true +[ const_inds ] + standard_name = test_banana_constituent_indices + long_name = Array of constituent indices + units = 1 + dimensions = (banana_array_dim) + protected = true + type = integer +[ const_index ] + standard_name = test_banana_constituent_index + long_name = Constituent index + units = 1 + dimensions = () + type = integer diff --git a/end-to-end-tests/advection_auto_clone/test_host_mod.F90 b/end-to-end-tests/advection_auto_clone/test_host_mod.F90 new file mode 100644 index 00000000..5099b9c1 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/test_host_mod.F90 @@ -0,0 +1,176 @@ +module test_host_mod + + use ccpp_kinds, only: kind_phys + use test_host_data, only: physics_state, & + allocate_physics_state + + implicit none + public + + integer, parameter :: num_time_steps = 2 + real(kind=kind_phys), parameter :: tolerance = 1.0e-13_kind_phys + + !> \section arg_table_test_host_mod Argument Table + !! \htmlinclude arg_table_test_host_mod.html + !! + integer, parameter :: ncols = 10 + integer, parameter :: pver = 5 + integer, parameter :: pverp = pver + 1 + integer, protected :: ncnst = -1 + integer, protected :: index_qv = -1 + real(kind=kind_phys) :: dt + real(kind=kind_phys), parameter :: tfreeze = 273.15_kind_phys + type(physics_state) :: phys_state + integer :: num_model_times = -1 + integer, allocatable :: model_times(:) + + public :: init_data + public :: compare_data + public :: twist_array + + real(kind=kind_phys), private, allocatable :: check_vals(:, :, :) + real(kind=kind_phys), private :: check_temp(ncols, pver) + integer, private :: ind_liq = -1 + integer, private :: ind_ice = -1 + +contains + + subroutine init_data(constituent_array, index_qv_use, index_liq, index_ice, index_dyn) + + ! Dummy arguments + real(kind=kind_phys), pointer :: constituent_array(:, :, :) ! From host & suites + integer, intent(in) :: index_qv_use + integer, intent(in) :: index_liq + integer, intent(in) :: index_ice + integer, intent(in) :: index_dyn + + ! Local variables + integer :: col + integer :: lev + integer :: cind + integer :: itime + real(kind=kind_phys) :: qmax + real(kind=kind_phys), parameter :: inc = 0.1_kind_phys + + ! Allocate and initialize state + ! Temperature starts above freezing and decreases to -30C + ! water vapor is initialized in odd columns to different amounts + ncnst = size(constituent_array, 3) + call allocate_physics_state(ncols, pver, constituent_array, phys_state) + index_qv = index_qv_use + ind_liq = index_liq + ind_ice = index_ice + allocate(check_vals(ncols, pver, ncnst)) + check_vals(:, :, :) = 0.0_kind_phys + check_vals(:, :, index_dyn) = 1.0_kind_phys + do lev = 1, pver + phys_state%temp(:, lev) = tfreeze + (10.0_kind_phys * (lev - 3)) + qmax = real(lev, kind_phys) + do col = 1, ncols + if (mod(col, 2) == 1) then + phys_state%q(col, lev, index_qv) = qmax + else + phys_state%q(col, lev, index_qv) = 0.0_kind_phys + end if + end do + end do + check_vals(:, :, index_qv) = phys_state%q(:, :, index_qv) + check_temp(:, :) = phys_state%temp(:, :) + ! Do timestep 1 + do col = 1, ncols, 2 + check_temp(col, 1) = check_temp(col, 1) + 0.5_kind_phys + check_vals(col, 1, index_qv) = check_vals(col, 1, index_qv) - inc + check_vals(col, 1, ind_liq) = check_vals(col, 1, ind_liq) + inc + end do + do itime = 1, num_time_steps + do cind = 1, ncnst + call twist_array(check_vals(:, :, cind)) + end do + end do + + end subroutine init_data + + subroutine twist_array(array) + ! Dummy argument + real(kind=kind_phys), intent(inout) :: array(:, :) + + ! Local variables + integer :: icol, ilev ! Field coordinates + integer :: idir ! 'w' sign + integer :: levb, leve ! Starting and ending level indices + real(kind=kind_phys) :: last_val, next_val + + idir = 1 + leve = (pver * mod(ncols, 2)) + mod(ncols - 1, 2) + last_val = array(ncols, leve) + do icol = 1, ncols + levb = ((pver * (1 - idir)) + (1 + idir)) / 2 + leve = ((pver * (1 + idir)) + (1 - idir)) / 2 + do ilev = levb, leve, idir + next_val = array(icol, ilev) + array(icol, ilev) = last_val + last_val = next_val + end do + idir = -1 * idir + end do + + end subroutine twist_array + + logical function compare_data(ncnst) + + integer, intent(in) :: ncnst + + integer :: col + integer :: lev + integer :: cind + logical :: need_header + real(kind=kind_phys) :: check + real(kind=kind_phys) :: denom + + compare_data = .true. + + need_header = .true. + do lev = 1, pver + do col = 1, ncols + check = check_temp(col, lev) + if (abs((phys_state%temp(col, lev) - check) / check) > & + tolerance) then + if (need_header) then + write(6, '(" COL LEV T MIDPOINTS EXPECTED")') + need_header = .false. + end if + write(6, '(2i5,2(3x,es15.7))') col, lev, & + phys_state%temp(col, lev), check + compare_data = .false. + end if + end do + end do + ! Check constituents + need_header = .true. + do cind = 1, ncnst + do lev = 1, pver + do col = 1, ncols + check = check_vals(col, lev, cind) + if (check < tolerance) then + denom = 1.0_kind_phys + else + denom = check + end if + if (abs((phys_state%q(col, lev, cind) - check) / denom) > & + tolerance) then + if (need_header) then + write(6, '(2(2x,a),3x,a,10x,a,14x,a)') & + 'COL', 'LEV', 'C#', 'Q', 'EXPECTED' + need_header = .false. + end if + write(6, '(3i5,2(3x,es15.7))') col, lev, cind, & + phys_state%q(col, lev, cind), check + compare_data = .false. + end if + end do + end do + end do + + end function compare_data + +end module test_host_mod diff --git a/end-to-end-tests/advection_auto_clone/test_host_mod.meta b/end-to-end-tests/advection_auto_clone/test_host_mod.meta new file mode 100644 index 00000000..6c3d15eb --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/test_host_mod.meta @@ -0,0 +1,64 @@ +[ccpp-table-properties] + name = test_host_mod + type = host +[ccpp-arg-table] + name = test_host_mod + type = host +[ ncols] + standard_name = horizontal_dimension + units = count + type = integer + protected = True + dimensions = () +[ pver ] + standard_name = vertical_layer_dimension + units = count + type = integer + protected = True + dimensions = () +[ pverP ] + standard_name = vertical_interface_dimension + type = integer + units = count + protected = True + dimensions = () +[ ncnst ] + standard_name = number_of_tracers + type = integer + units = count + protected = True + dimensions = () +[ index_qv ] + standard_name = index_of_water_vapor_specific_humidity + units = index + type = integer + protected = True + dimensions = () +[ dt ] + standard_name = time_step_for_physics + long_name = time step + units = s + dimensions = () + type = real | kind = kind_phys +[ tfreeze ] + standard_name = water_temperature_at_freezing + long_name = Freezing temperature of water at sea level + units = K + dimensions = () + type = real | kind = kind_phys +[ phys_state ] + standard_name = physics_state_derived_type + long_name = Physics State DDT + type = physics_state + dimensions = () +[ num_model_times ] + standard_name = number_of_model_times + type = integer + units = count + dimensions = () +[ model_times ] + standard_name = model_times + units = seconds + dimensions = (number_of_model_times) + type = integer + allocatable = True diff --git a/end-to-end-tests/capgen/CMakeLists.txt b/end-to-end-tests/capgen/CMakeLists.txt new file mode 100644 index 00000000..4df45d1e --- /dev/null +++ b/end-to-end-tests/capgen/CMakeLists.txt @@ -0,0 +1,99 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "setup_coeffs" "temp_set" "temp_adjust" "temp_calc_adjust" "make_ddt" "environ_conditions") +set(HOST_FILES "test_host_data" "test_host_mod" "test_host") +set(SUITE_FILES "ddt_suite.xml" "temp_suite.xml") +set(HOST "test_host") +set(KIND_TYPE "kind_phys=REAL64") +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +# Fortran files are not all in one directory +set(SCHEME_FORTRAN_FILES "") +foreach(sfile ${SCHEME_FILES}) + find_file(fort_file "${sfile}.F90" NO_CACHE + HINTS ${CMAKE_CURRENT_SOURCE_DIR} + HINTS ${CMAKE_CURRENT_SOURCE_DIR}/source_dir1 + HINTS ${CMAKE_CURRENT_SOURCE_DIR}/source_dir2 + HINTS ${CMAKE_CURRENT_SOURCE_DIR}/adjust) + list(APPEND SCHEME_FORTRAN_FILES ${fort_file}) + unset(fort_file) +endforeach() +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") + +# Run ccpp_capgen +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +set(EXTRA_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/../utils/test_utils.F90 +) + +add_executable(test_capgen.x + ${EXTRA_FILES} + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES_FILTERED} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES} + test_capgen_host_integration.F90 +) +target_link_libraries(test_capgen.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_capgen.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_capgen.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_capgen_omp1 + COMMAND test_capgen.x) + +add_test(NAME test_capgen_omp2 + COMMAND test_capgen.x) + +set_tests_properties(test_capgen_omp1 + PROPERTIES + ENVIRONMENT "OMP_NUM_THREADS=1" +) + +set_tests_properties(test_capgen_omp2 + PROPERTIES + ENVIRONMENT "OMP_NUM_THREADS=2" +) diff --git a/test/capgen_test/README.md b/end-to-end-tests/capgen/README.md similarity index 69% rename from test/capgen_test/README.md rename to end-to-end-tests/capgen/README.md index bcc0e628..a56bfcdf 100644 --- a/test/capgen_test/README.md +++ b/end-to-end-tests/capgen/README.md @@ -10,14 +10,5 @@ Contains tests for overall capgen capabilities such as: - Variables that should be promoted to suite level - Dimensions that are set in the register phase and used to allocate module-level interstitial variables +- Threading -## Building/Running - -To explicitly build/run the capgen test host, run: - -```bash -$ cmake --fresh -S -B -DCCPP_RUN_CAPGEN_TEST=ON -$ cd -$ make -$ ctest -``` diff --git a/test/capgen_test/adjust/temp_kinds.F90 b/end-to-end-tests/capgen/adjust/temp_kinds.F90 similarity index 100% rename from test/capgen_test/adjust/temp_kinds.F90 rename to end-to-end-tests/capgen/adjust/temp_kinds.F90 diff --git a/test/capgen_test/capgen_test_reports.py b/end-to-end-tests/capgen/capgen_test_reports.py similarity index 98% rename from test/capgen_test/capgen_test_reports.py rename to end-to-end-tests/capgen/capgen_test_reports.py index 3c683aab..c168827d 100644 --- a/test/capgen_test/capgen_test_reports.py +++ b/end-to-end-tests/capgen/capgen_test_reports.py @@ -36,8 +36,7 @@ [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90"), os.path.join(_BUILD_DIR, "ccpp", "ccpp_ddt_suite_cap.F90"), os.path.join(_BUILD_DIR, "ccpp", "ccpp_temp_suite_cap.F90")] -_DEPENDENCIES = [os.path.join(_TEST_DIR, "adjust", "qux.F90"), - os.path.join(_TEST_DIR, "adjust", "temp_kinds.F90"), +_DEPENDENCIES = [os.path.join(_TEST_DIR, "adjust", "temp_kinds.F90"), os.path.join(_TEST_DIR, "ddt2"), os.path.join(_TEST_DIR, "bar.F90"), os.path.join(_TEST_DIR, "foo.F90")] diff --git a/test/capgen_test/ddt2.F90 b/end-to-end-tests/capgen/ddt2.F90 similarity index 100% rename from test/capgen_test/ddt2.F90 rename to end-to-end-tests/capgen/ddt2.F90 diff --git a/test/ddthost_test/ddt_suite.xml b/end-to-end-tests/capgen/ddt_suite.xml similarity index 80% rename from test/ddthost_test/ddt_suite.xml rename to end-to-end-tests/capgen/ddt_suite.xml index 749bb3bc..72c9c436 100644 --- a/test/ddthost_test/ddt_suite.xml +++ b/end-to-end-tests/capgen/ddt_suite.xml @@ -1,6 +1,6 @@ - + make_ddt environ_conditions diff --git a/test/capgen_test/environ_conditions.meta b/end-to-end-tests/capgen/environ_conditions.meta similarity index 95% rename from test/capgen_test/environ_conditions.meta rename to end-to-end-tests/capgen/environ_conditions.meta index f87e5039..30693cf7 100644 --- a/test/capgen_test/environ_conditions.meta +++ b/end-to-end-tests/capgen/environ_conditions.meta @@ -7,11 +7,10 @@ type = scheme [ psurf ] standard_name = surface_air_pressure - state_variable = true type = real kind = kind_phys units = Pa - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) intent = in [ errmsg ] standard_name = ccpp_error_message @@ -80,7 +79,7 @@ type = integer intent = out [ccpp-arg-table] - name = environ_conditions_finalize + name = environ_conditions_final type = scheme [ ntimes ] standard_name = number_of_model_times diff --git a/test/capgen_test/make_ddt.F90 b/end-to-end-tests/capgen/make_ddt.F90 similarity index 100% rename from test/capgen_test/make_ddt.F90 rename to end-to-end-tests/capgen/make_ddt.F90 diff --git a/test/capgen_test/make_ddt.meta b/end-to-end-tests/capgen/make_ddt.meta similarity index 96% rename from test/capgen_test/make_ddt.meta rename to end-to-end-tests/capgen/make_ddt.meta index 2f3eeaa5..5981c26a 100644 --- a/test/capgen_test/make_ddt.meta +++ b/end-to-end-tests/capgen/make_ddt.meta @@ -1,7 +1,7 @@ [ccpp-table-properties] name = vmr_type type = ddt - dependencies = ddt2 + dependencies = ddt2.F90 [ccpp-arg-table] name = vmr_type type = ddt @@ -37,14 +37,14 @@ [ O3 ] standard_name = ozone units = ppmv - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) type = real kind = kind_phys intent = in [ HNO3 ] standard_name = nitric_acid units = ppmv - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) type = real kind = kind_phys intent = in diff --git a/test/capgen_test/setup_coeffs.F90 b/end-to-end-tests/capgen/setup_coeffs.F90 similarity index 100% rename from test/capgen_test/setup_coeffs.F90 rename to end-to-end-tests/capgen/setup_coeffs.F90 diff --git a/test/capgen_test/setup_coeffs.meta b/end-to-end-tests/capgen/setup_coeffs.meta similarity index 100% rename from test/capgen_test/setup_coeffs.meta rename to end-to-end-tests/capgen/setup_coeffs.meta diff --git a/test/ddthost_test/environ_conditions.F90 b/end-to-end-tests/capgen/source_dir1/environ_conditions.F90 similarity index 90% rename from test/ddthost_test/environ_conditions.F90 rename to end-to-end-tests/capgen/source_dir1/environ_conditions.F90 index 2d63366e..9c92faea 100644 --- a/test/ddthost_test/environ_conditions.F90 +++ b/end-to-end-tests/capgen/source_dir1/environ_conditions.F90 @@ -7,7 +7,7 @@ module environ_conditions public :: environ_conditions_init public :: environ_conditions_run - public :: environ_conditions_finalize + public :: environ_conditions_final integer, parameter :: input_model_times = 3 integer, parameter :: input_model_values(input_model_times) = (/ 31, 37, 41 /) @@ -63,10 +63,10 @@ subroutine environ_conditions_init(nbox, o3, hno3, ntimes, model_times, & end subroutine environ_conditions_init - !> \section arg_table_environ_conditions_finalize Argument Table - !! \htmlinclude arg_table_environ_conditions_finalize.html + !> \section arg_table_environ_conditions_final Argument Table + !! \htmlinclude arg_table_environ_conditions_final.html !! - subroutine environ_conditions_finalize(ntimes, model_times, errmsg, errflg) + subroutine environ_conditions_final(ntimes, model_times, errmsg, errflg) integer, intent(in) :: ntimes integer, intent(in) :: model_times(:) @@ -91,6 +91,6 @@ subroutine environ_conditions_finalize(ntimes, model_times, errmsg, errflg) errflg = 0 end if - end subroutine environ_conditions_finalize + end subroutine environ_conditions_final end module environ_conditions diff --git a/test/capgen_test/source_dir2/temp_set.F90 b/end-to-end-tests/capgen/source_dir2/temp_set.F90 similarity index 81% rename from test/capgen_test/source_dir2/temp_set.F90 rename to end-to-end-tests/capgen/source_dir2/temp_set.F90 index be54b80c..30e0ec2a 100644 --- a/test/capgen_test/source_dir2/temp_set.F90 +++ b/end-to-end-tests/capgen/source_dir2/temp_set.F90 @@ -10,9 +10,9 @@ module temp_set private public :: temp_set_init - public :: temp_set_timestep_initialize + public :: temp_set_timestep_init public :: temp_set_run - public :: temp_set_finalize + public :: temp_set_final contains @@ -72,10 +72,11 @@ end subroutine temp_set_run !> \section arg_table_temp_set_init Argument Table !! \htmlinclude arg_table_temp_set_init.html !! - subroutine temp_set_init(temp_inc_in, fudge, temp_inc_set, errmsg, errflg) + !subroutine temp_set_init(temp_inc_in, fudge, temp_inc_set, errmsg, errflg) + subroutine temp_set_init(temp_inc_in, temp_inc_set, errmsg, errflg) real(kind=kind_phys), intent(in) :: temp_inc_in - real(kind=kind_phys), intent(in) :: fudge + !real(kind=kind_phys), intent(in) :: fudge real(kind=kind_phys), intent(out) :: temp_inc_set character(len=512), intent(out) :: errmsg integer, intent(out) :: errflg @@ -87,10 +88,10 @@ subroutine temp_set_init(temp_inc_in, fudge, temp_inc_set, errmsg, errflg) end subroutine temp_set_init - !> \section arg_table_temp_set_timestep_initialize Argument Table - !! \htmlinclude arg_table_temp_set_timestep_initialize.html + !> \section arg_table_temp_set_timestep_init Argument Table + !! \htmlinclude arg_table_temp_set_timestep_init.html !! - subroutine temp_set_timestep_initialize(ncol, temp_inc, temp_level, & + subroutine temp_set_timestep_init(ncol, temp_inc, temp_level, & errmsg, errflg) integer, intent(in) :: ncol @@ -104,12 +105,12 @@ subroutine temp_set_timestep_initialize(ncol, temp_inc, temp_level, & temp_level = temp_level + temp_inc - end subroutine temp_set_timestep_initialize + end subroutine temp_set_timestep_init - !> \section arg_table_temp_set_finalize Argument Table - !! \htmlinclude arg_table_temp_set_finalize.html + !> \section arg_table_temp_set_final Argument Table + !! \htmlinclude arg_table_temp_set_final.html !! - subroutine temp_set_finalize(errmsg, errflg) + subroutine temp_set_final(errmsg, errflg) character(len=512), intent(out) :: errmsg integer, intent(out) :: errflg @@ -119,6 +120,6 @@ subroutine temp_set_finalize(errmsg, errflg) errmsg = '' errflg = 0 - end subroutine temp_set_finalize + end subroutine temp_set_final end module temp_set diff --git a/test/capgen_test/temp_adjust.F90 b/end-to-end-tests/capgen/temp_adjust.F90 similarity index 92% rename from test/capgen_test/temp_adjust.F90 rename to end-to-end-tests/capgen/temp_adjust.F90 index e8ac281d..515cf4a8 100644 --- a/test/capgen_test/temp_adjust.F90 +++ b/end-to-end-tests/capgen/temp_adjust.F90 @@ -12,7 +12,7 @@ module temp_adjust public :: temp_adjust_register public :: temp_adjust_init public :: temp_adjust_run - public :: temp_adjust_finalize + public :: temp_adjust_final logical :: module_level_config = .false. @@ -99,10 +99,10 @@ subroutine temp_adjust_init(errmsg, errflg) end subroutine temp_adjust_init - !> \section arg_table_temp_adjust_finalize Argument Table - !! \htmlinclude arg_table_temp_adjust_finalize.html + !> \section arg_table_temp_adjust_final Argument Table + !! \htmlinclude arg_table_temp_adjust_final.html !! - subroutine temp_adjust_finalize(interstitial_var, errmsg, errflg) + subroutine temp_adjust_final(interstitial_var, errmsg, errflg) integer, intent(in) :: interstitial_var(:) character(len=512), intent(out) :: errmsg @@ -122,6 +122,6 @@ subroutine temp_adjust_finalize(interstitial_var, errmsg, errflg) errmsg = 'interstitial variable not set properly!' end if - end subroutine temp_adjust_finalize + end subroutine temp_adjust_final end module temp_adjust diff --git a/test/capgen_test/temp_adjust.meta b/end-to-end-tests/capgen/temp_adjust.meta similarity index 88% rename from test/capgen_test/temp_adjust.meta rename to end-to-end-tests/capgen/temp_adjust.meta index 63e7fcc1..8f53ad76 100644 --- a/test/capgen_test/temp_adjust.meta +++ b/end-to-end-tests/capgen/temp_adjust.meta @@ -2,7 +2,7 @@ name = temp_adjust type = scheme kind_spec = temp_kinds:kind_temp=>temp_r8 - dependencies = qux.F90, temp_kinds.F90 + dependencies = temp_kinds.F90 dependencies_path = adjust [ccpp-arg-table] name = temp_adjust_register @@ -32,7 +32,7 @@ name = temp_adjust_run type = scheme [ foo ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension type = integer units = count dimensions = () @@ -54,14 +54,14 @@ [ temp_prev ] standard_name = potential_temperature_at_previous_timestep units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = in [ temp_layer ] standard_name = potential_temperature units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = inout @@ -69,7 +69,7 @@ [ qv ] standard_name = water_vapor_specific_humidity units = kg kg-1 - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = inout @@ -77,16 +77,15 @@ optional = True [ ps ] standard_name = surface_air_pressure - state_variable = true type = real kind = kind_phys units = Pa - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) intent = inout [ to_promote ] standard_name = promote_this_variable_to_suite units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_temp intent = in @@ -131,7 +130,7 @@ type = integer intent = out [ccpp-arg-table] - name = temp_adjust_finalize + name = temp_adjust_final type = scheme [ interstitial_var ] standard_name = output_only_interstitial_variable diff --git a/test/capgen_test/temp_calc_adjust.F90 b/end-to-end-tests/capgen/temp_calc_adjust.F90 similarity index 90% rename from test/capgen_test/temp_calc_adjust.F90 rename to end-to-end-tests/capgen/temp_calc_adjust.F90 index 7c423669..54312133 100644 --- a/test/capgen_test/temp_calc_adjust.F90 +++ b/end-to-end-tests/capgen/temp_calc_adjust.F90 @@ -11,7 +11,7 @@ module temp_calc_adjust public :: temp_calc_adjust_register public :: temp_calc_adjust_init public :: temp_calc_adjust_run - public :: temp_calc_adjust_finalize + public :: temp_calc_adjust_final contains @@ -93,10 +93,10 @@ subroutine temp_calc_adjust_init(errmsg, errflg) end subroutine temp_calc_adjust_init - !> \section arg_table_temp_calc_adjust_finalize Argument Table - !! \htmlinclude arg_table_temp_calc_adjust_finalize.html + !> \section arg_table_temp_calc_adjust_final Argument Table + !! \htmlinclude arg_table_temp_calc_adjust_final.html !! - subroutine temp_calc_adjust_finalize(errmsg, errflg) + subroutine temp_calc_adjust_final(errmsg, errflg) character(len=512), intent(out) :: errmsg integer, intent(out) :: errflg @@ -106,6 +106,6 @@ subroutine temp_calc_adjust_finalize(errmsg, errflg) errmsg = '' errflg = 0 - end subroutine temp_calc_adjust_finalize + end subroutine temp_calc_adjust_final end module temp_calc_adjust diff --git a/test/capgen_test/temp_calc_adjust.meta b/end-to-end-tests/capgen/temp_calc_adjust.meta similarity index 89% rename from test/capgen_test/temp_calc_adjust.meta rename to end-to-end-tests/capgen/temp_calc_adjust.meta index f795da63..4a959c42 100644 --- a/test/capgen_test/temp_calc_adjust.meta +++ b/end-to-end-tests/capgen/temp_calc_adjust.meta @@ -1,7 +1,7 @@ [ccpp-table-properties] name = temp_calc_adjust type = scheme - dependencies = foo.F90, bar.F90 + dependencies = [ccpp-arg-table] name = temp_calc_adjust_register type = scheme @@ -31,7 +31,7 @@ type = scheme process = adjusting [ nbox ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension type = integer units = count dimensions = () @@ -47,14 +47,14 @@ [ temp_level ] standard_name = potential_temperature_at_interface units = K - dimensions = (ccpp_constant_one:horizontal_loop_extent, vertical_interface_dimension) + dimensions = (ccpp_constant_one:horizontal_dimension, vertical_interface_dimension) type = real kind = kind_phys intent = in [ temp_calc ] standard_name = potential_temperature_at_previous_timestep units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = out @@ -92,7 +92,7 @@ type = integer intent = out [ccpp-arg-table] - name = temp_calc_adjust_finalize + name = temp_calc_adjust_final type = scheme [ errmsg ] standard_name = ccpp_error_message diff --git a/test/capgen_test/temp_set.meta b/end-to-end-tests/capgen/temp_set.meta similarity index 86% rename from test/capgen_test/temp_set.meta rename to end-to-end-tests/capgen/temp_set.meta index 42bbb194..f6cdec65 100644 --- a/test/capgen_test/temp_set.meta +++ b/end-to-end-tests/capgen/temp_set.meta @@ -10,7 +10,7 @@ type = scheme process = setter [ ncol ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension type = integer units = count dimensions = () @@ -32,36 +32,35 @@ [ temp_level ] standard_name = potential_temperature_at_interface units = K - dimensions = (ccpp_constant_one:horizontal_loop_extent, vertical_interface_dimension) + dimensions = (ccpp_constant_one:horizontal_dimension, vertical_interface_dimension) type = real kind = kind_phys intent = inout [ temp_diag ] standard_name = temperature_at_diagnostic_levels units = K - dimensions = (horizontal_loop_extent, 6) + dimensions = (horizontal_dimension, 6) type = real kind = kind_phys intent = inout [ temp ] standard_name = potential_temperature units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = out [ ps ] standard_name = surface_air_pressure - state_variable = true type = real kind = kind_phys units = Pa - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) intent = in [ to_promote ] standard_name = promote_this_variable_to_suite units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_temp intent = out @@ -90,7 +89,7 @@ standard_name = array_variable_for_testing long_name = array variable for testing units = none - dimensions = (horizontal_loop_extent,2,4,6) + dimensions = (horizontal_dimension,2,4,6) type = real kind = kind_phys intent = inout @@ -121,15 +120,15 @@ type = real kind = kind_phys intent = in -[ fudge ] - standard_name = random_fudge_factor - long_name = Ignore this - units = 1 - dimensions = () - type = real - kind = kind_phys - intent = in - default_value = 1.0_kind_phys +#[ fudge ] +# standard_name = random_fudge_factor +# long_name = Ignore this +# units = 1 +# dimensions = () +# type = real +# kind = kind_phys +# intent = in +# default_value = 1.0_kind_phys [ temp_inc_set ] standard_name = test_potential_temperature_increment long_name = Per time step potential temperature increment @@ -155,7 +154,7 @@ intent = out # Timestep Initialization [ccpp-arg-table] - name = temp_set_timestep_initialize + name = temp_set_timestep_init type = scheme [ ncol ] standard_name = horizontal_dimension @@ -195,7 +194,7 @@ intent = out # Finalize [ccpp-arg-table] - name = temp_set_finalize + name = temp_set_final type = scheme [ errmsg ] standard_name = ccpp_error_message diff --git a/test/capgen_test/temp_suite.xml b/end-to-end-tests/capgen/temp_suite.xml similarity index 100% rename from test/capgen_test/temp_suite.xml rename to end-to-end-tests/capgen/temp_suite.xml diff --git a/end-to-end-tests/capgen/test_capgen_host_integration.F90 b/end-to-end-tests/capgen/test_capgen_host_integration.F90 new file mode 100644 index 00000000..9dc43c89 --- /dev/null +++ b/end-to-end-tests/capgen/test_capgen_host_integration.F90 @@ -0,0 +1,91 @@ +program test + use test_prog, only: test_host, & + suite_info, & + cm, & + cs + + implicit none + + character(len=cs), target :: test_parts1(2) = (/ 'physics1 ', & + 'physics2 ' /) + character(len=cs), target :: test_parts2(1) = (/ 'data_prep ' /) + character(len=cm), target :: test_invars1(11) = (/ & + 'potential_temperature ', & + 'potential_temperature_at_interface ', & + 'coefficients_for_interpolation ', & + 'index_of_water_vapor_specific_humidity', & + 'surface_air_pressure ', & + 'water_vapor_specific_humidity ', & + 'potential_temperature_increment ', & + 'soil_levels ', & + 'temperature_at_diagnostic_levels ', & + 'time_step_for_physics ', & + 'array_variable_for_testing ' /) + character(len=cm), target :: test_outvars1(10) = (/ & + 'potential_temperature ', & + 'potential_temperature_at_interface ', & + 'coefficients_for_interpolation ', & + 'surface_air_pressure ', & + 'water_vapor_specific_humidity ', & + 'soil_levels ', & + 'temperature_at_diagnostic_levels ', & + 'ccpp_error_code ', & + 'ccpp_error_message ', & + 'array_variable_for_testing ' /) + character(len=cm), target :: test_reqvars1(13) = (/ & + 'potential_temperature ', & + 'potential_temperature_at_interface ', & + 'coefficients_for_interpolation ', & + 'index_of_water_vapor_specific_humidity', & + 'surface_air_pressure ', & + 'water_vapor_specific_humidity ', & + 'potential_temperature_increment ', & + 'time_step_for_physics ', & + 'soil_levels ', & + 'temperature_at_diagnostic_levels ', & + 'ccpp_error_code ', & + 'ccpp_error_message ', & + 'array_variable_for_testing ' /) + + character(len=cm), target :: test_invars2(3) = (/ & + 'model_times ', & + 'number_of_model_times ', & + 'surface_air_pressure ' /) + + character(len=cm), target :: test_outvars2(4) = (/ & + 'ccpp_error_code ', & + 'ccpp_error_message ', & + 'model_times ', & + 'number_of_model_times ' /) + + character(len=cm), target :: test_reqvars2(5) = (/ & + 'ccpp_error_code ', & + 'ccpp_error_message ', & + 'model_times ', & + 'number_of_model_times ', & + 'surface_air_pressure ' /) + + type(suite_info) :: test_suites(2) + logical :: run_okay + + ! Setup expected test suite info + test_suites(1)%suite_name = 'temp_suite' + test_suites(1)%suite_parts => test_parts1 + test_suites(1)%suite_input_vars => test_invars1 + test_suites(1)%suite_output_vars => test_outvars1 + test_suites(1)%suite_required_vars => test_reqvars1 + test_suites(2)%suite_name = 'ddt_suite' + test_suites(2)%suite_parts => test_parts2 + test_suites(2)%suite_input_vars => test_invars2 + test_suites(2)%suite_output_vars => test_outvars2 + test_suites(2)%suite_required_vars => test_reqvars2 + + call test_host(run_okay, test_suites) + + if (run_okay) then + stop 0 + else + stop -1 + end if + +end program test diff --git a/test/capgen_test/test_host.F90 b/end-to-end-tests/capgen/test_host.F90 similarity index 73% rename from test/capgen_test/test_host.F90 rename to end-to-end-tests/capgen/test_host.F90 index 258f0d91..43f01aa0 100644 --- a/test/capgen_test/test_host.F90 +++ b/end-to-end-tests/capgen/test_host.F90 @@ -9,7 +9,7 @@ module test_prog ! Public data and interfaces integer, public, parameter :: cs = 16 - integer, public, parameter :: cm = 36 + integer, public, parameter :: cm = 64 !> \section arg_table_suite_info Argument Table !! \htmlinclude arg_table_suite_info.html @@ -108,12 +108,14 @@ subroutine test_host(retval, test_suites) #endif use test_host_mod, only: ncols, & num_time_steps - use test_host_ccpp_cap, only: test_host_ccpp_physics_register - use test_host_ccpp_cap, only: test_host_ccpp_physics_initialize - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_initial - use test_host_ccpp_cap, only: test_host_ccpp_physics_run - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_final - use test_host_ccpp_cap, only: test_host_ccpp_physics_finalize + use test_host_ccpp_cap, only: ccpp_register + use test_host_ccpp_cap, only: ccpp_init + use test_host_ccpp_cap, only: ccpp_physics_init + use test_host_ccpp_cap, only: ccpp_physics_timestep_init + use test_host_ccpp_cap, only: ccpp_physics_run + use test_host_ccpp_cap, only: ccpp_physics_timestep_final + use test_host_ccpp_cap, only: ccpp_physics_final + use test_host_ccpp_cap, only: ccpp_final use test_host_ccpp_cap, only: ccpp_physics_suite_list use test_host_mod, only: init_data, & compare_data, & @@ -165,19 +167,31 @@ subroutine test_host(retval, test_suites) ! Use the suite information to call the register phase do sind = 1, num_suites - call test_host_ccpp_physics_register(test_suites(sind)%suite_name, & - errmsg, errflg) + call ccpp_register(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) if (errflg /= 0) then - write(6, '(4a)') 'ERROR in register of ', & + write(6, '(4a)') 'ERROR in ccpp_register for ', & trim(test_suites(sind)%suite_name), ': ', trim(errmsg) end if end do + ! Call the CCPP init phase for each suite + do sind = 1, num_suites + call ccpp_init(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in ccpp_init for ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + end if + end do ! Use the suite information to setup the run do sind = 1, num_suites - call test_host_ccpp_physics_initialize(test_suites(sind)%suite_name, & - errmsg, errflg) + call ccpp_physics_init( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1) if (errflg /= 0) then - write(6, '(4a)') 'ERROR in initialize of ', & + write(6, '(4a)') 'ERROR in ccpp_physics_init for ', & trim(test_suites(sind)%suite_name), ': ', trim(errmsg) end if end do @@ -189,8 +203,11 @@ subroutine test_host(retval, test_suites) exit end if if (errflg == 0) then - call test_host_ccpp_physics_timestep_initial( & - test_suites(sind)%suite_name, errmsg, errflg) + call ccpp_physics_timestep_init( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1) end if if (errflg /= 0) then write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & @@ -236,10 +253,13 @@ subroutine test_host(retval, test_suites) ': calling run phase for suite ', trim(test_suites(sind)%suite_name), & ' part ', trim(test_suites(sind)%suite_parts(index)), & ' columns ', col_start, ':', col_end - call test_host_ccpp_physics_run( & - test_suites(sind)%suite_name, & - test_suites(sind)%suite_parts(index), & - col_start, col_end, errmsg, errflg) + call ccpp_physics_run( & + suite_name=test_suites(sind)%suite_name, & + group_name=test_suites(sind)%suite_parts(index), & + col_start=col_start, col_end=col_end, & + errmsg=errmsg, errflg=errflg, & + thread_num=thread_num, nthreads=num_threads, & + nphys_threads=1) if (errflg /= 0) then write(6, '(5a)') trim(test_suites(sind)%suite_name), & '/', trim(test_suites(sind)%suite_parts(index)), & @@ -257,8 +277,11 @@ subroutine test_host(retval, test_suites) exit end if if (errflg == 0) then - call test_host_ccpp_physics_timestep_final( & - test_suites(sind)%suite_name, errmsg, errflg) + call ccpp_physics_timestep_final( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1) end if if (errflg /= 0) then write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & @@ -273,13 +296,33 @@ subroutine test_host(retval, test_suites) exit end if if (errflg == 0) then - call test_host_ccpp_physics_finalize( & - test_suites(sind)%suite_name, errmsg, errflg) + call ccpp_physics_final( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1) + end if + if (errflg /= 0) then + write(6, '(3a)') test_suites(sind)%suite_name, ': ', & + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_physics_final, ', & + 'Exiting...' + exit + end if + end do + + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_final(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) end if if (errflg /= 0) then - write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & + write(6, '(3a)') test_suites(sind)%suite_name, ': ', & trim(errmsg) - write(6, '(2a)') 'An error occurred in ccpp_timestep_final, ', & + write(6, '(2a)') 'An error occurred in ccpp_final, ', & 'Exiting...' exit end if diff --git a/end-to-end-tests/capgen/test_host.meta b/end-to-end-tests/capgen/test_host.meta new file mode 100644 index 00000000..ab33172f --- /dev/null +++ b/end-to-end-tests/capgen/test_host.meta @@ -0,0 +1,70 @@ +[ccpp-table-properties] + name = suite_info + type = ddt +[ccpp-arg-table] + name = suite_info + type = ddt + +[ccpp-table-properties] + name = test_host + type = control +[ccpp-arg-table] + name = test_host + type = control +[ col_start ] + standard_name = horizontal_loop_begin + type = integer + units = count + dimensions = () + protected = True +[ col_end ] + standard_name = horizontal_loop_end + type = integer + units = count + dimensions = () + protected = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer diff --git a/test/capgen_test/test_host_data.F90 b/end-to-end-tests/capgen/test_host_data.F90 similarity index 93% rename from test/capgen_test/test_host_data.F90 rename to end-to-end-tests/capgen/test_host_data.F90 index 32c421a4..3d332af3 100644 --- a/test/capgen_test/test_host_data.F90 +++ b/end-to-end-tests/capgen/test_host_data.F90 @@ -15,7 +15,7 @@ module test_host_data u, & ! zonal wind (m/s) v, & ! meridional wind (m/s) pmid ! midpoint pressure (Pa) - real(kind=kind_phys), dimension(:, :, :), allocatable :: & + real(kind=kind_phys), dimension(:, :, :), pointer :: & q ! constituent mixing ratio (kg/kg moist or dry air depending on type) end type physics_state @@ -47,8 +47,8 @@ subroutine allocate_physics_state(cols, levels, constituents, lbnd_slev, ubnd_sl deallocate(state%pmid) end if allocate(state%pmid(cols, levels)) - if (allocated(state%q)) then - deallocate(state%q) + if (associated(state%q)) then + nullify(state%q) end if allocate(state%q(cols, levels, constituents)) if (allocated(state%soil_levs)) then diff --git a/test/capgen_test/test_host_data.meta b/end-to-end-tests/capgen/test_host_data.meta similarity index 91% rename from test/capgen_test/test_host_data.meta rename to end-to-end-tests/capgen/test_host_data.meta index 0e73c060..aab2ce4a 100644 --- a/test/capgen_test/test_host_data.meta +++ b/end-to-end-tests/capgen/test_host_data.meta @@ -6,7 +6,6 @@ type = ddt [ ps ] standard_name = surface_air_pressure - state_variable = true type = real kind = kind_phys units = Pa @@ -14,7 +13,6 @@ [ u ] standard_name = eastward_wind long_name = Zonal wind - state_variable = true type = real kind = kind_phys units = m s-1 @@ -22,7 +20,6 @@ [ v ] standard_name = northward_wind long_name = Meridional wind - state_variable = true type = real kind = kind_phys units = m s-1 @@ -30,7 +27,6 @@ [ pmid ] standard_name = air_pressure long_name = Midpoint air pressure - state_variable = true type = real kind = kind_phys units = Pa @@ -44,14 +40,12 @@ kind = kind_phys [ q ] standard_name = constituent_mixing_ratio - state_variable = true type = real kind = kind_phys units = kg kg-1 moist or dry air depending on type dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_tracers) [ q(:,:,index_of_water_vapor_specific_HUMidity) ] standard_name = water_vapor_specific_humidity - state_variable = true type = real kind = kind_phys units = kg kg-1 diff --git a/test/capgen_test/test_host_mod.F90 b/end-to-end-tests/capgen/test_host_mod.F90 similarity index 100% rename from test/capgen_test/test_host_mod.F90 rename to end-to-end-tests/capgen/test_host_mod.F90 diff --git a/test/capgen_test/test_host_mod.meta b/end-to-end-tests/capgen/test_host_mod.meta similarity index 99% rename from test/capgen_test/test_host_mod.meta rename to end-to-end-tests/capgen/test_host_mod.meta index 08627af0..b92ce89a 100644 --- a/test/capgen_test/test_host_mod.meta +++ b/end-to-end-tests/capgen/test_host_mod.meta @@ -1,9 +1,9 @@ [ccpp-table-properties] name = test_host_mod - type = module + type = host [ccpp-arg-table] name = test_host_mod - type = module + type = host [ index_qv ] standard_name = index_of_water_vapor_specific_HUMidity units = index diff --git a/end-to-end-tests/chunked_data/CMakeLists.txt b/end-to-end-tests/chunked_data/CMakeLists.txt new file mode 100644 index 00000000..2bfef4f4 --- /dev/null +++ b/end-to-end-tests/chunked_data/CMakeLists.txt @@ -0,0 +1,69 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "chunked_data_scheme") +set(HOST_FILES "data" "main") +set(SUITE_FILES "suite_chunked_data_suite.xml") +set(HOST "test_host") +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") + +# Run ccpp_capgen +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +add_executable(test_chunked_data.x + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES_FILTERED} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES} +) +target_link_libraries(test_chunked_data.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_chunked_data.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_chunked_data.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_chunked_data + COMMAND test_chunked_data.x) diff --git a/test_prebuild/test_chunked_data/chunked_data_scheme.F90 b/end-to-end-tests/chunked_data/chunked_data_scheme.F90 similarity index 76% rename from test_prebuild/test_chunked_data/chunked_data_scheme.F90 rename to end-to-end-tests/chunked_data/chunked_data_scheme.F90 index 392167b2..e9d222ba 100644 --- a/test_prebuild/test_chunked_data/chunked_data_scheme.F90 +++ b/end-to-end-tests/chunked_data/chunked_data_scheme.F90 @@ -11,8 +11,8 @@ module chunked_data_scheme public :: chunked_data_scheme_init, & chunked_data_scheme_timestep_init, & chunked_data_scheme_run, & - chunked_data_scheme_timestep_finalize, & - chunked_data_scheme_finalize + chunked_data_scheme_timestep_final, & + chunked_data_scheme_final ! This is for unit testing only integer, parameter, dimension(4) :: data_array_sizes = (/6, 6, 6, 3/) @@ -63,28 +63,26 @@ end subroutine chunked_data_scheme_timestep_init !! \section arg_table_chunked_data_scheme_run Argument Table !! \htmlinclude chunked_data_scheme_run.html !! - subroutine chunked_data_scheme_run(nchunk, data_array, errmsg, errflg) + subroutine chunked_data_scheme_run(data_array, errmsg, errflg) character(len=*), intent(out) :: errmsg integer, intent(out) :: errflg - integer, intent(in) :: nchunk integer, intent(in) :: data_array(:) ! Initialize CCPP error handling variables errmsg = '' errflg = 0 - ! Check size of data array - write(error_unit, '(2(a,i3))') 'In chunked_data_scheme_run: checking size of data array for chunk', nchunk, & - ' to be', data_array_sizes(nchunk) - if (size(data_array)/=data_array_sizes(nchunk)) then - write(errmsg, '(a,i4)') "Error in chunked_data_scheme_run, expected size(data_array)==6, got ", size(data_array) + ! Check size of data array slice + write(error_unit, '(a,i3)') 'In chunked_data_scheme_run: checking size of data array slice', size(data_array) + if (.not. any(size(data_array)==data_array_sizes)) then + write(errmsg, '(a,i4)') "Error in chunked_data_scheme_run, unexpected size(data_array)=", size(data_array) errflg = 1 return end if end subroutine chunked_data_scheme_run - !! \section arg_table_chunked_data_scheme_timestep_finalize Argument Table - !! \htmlinclude chunked_data_scheme_timestep_finalize.html + !! \section arg_table_chunked_data_scheme_timestep_final Argument Table + !! \htmlinclude chunked_data_scheme_timestep_final.html !! - subroutine chunked_data_scheme_timestep_finalize(data_array, errmsg, errflg) + subroutine chunked_data_scheme_timestep_final(data_array, errmsg, errflg) character(len=*), intent(out) :: errmsg integer, intent(out) :: errflg integer, intent(in) :: data_array(:) @@ -92,7 +90,7 @@ subroutine chunked_data_scheme_timestep_finalize(data_array, errmsg, errflg) errmsg = '' errflg = 0 ! Check size of data array - write(error_unit, '(a,i3)') 'In chunked_data_scheme_timestep_finalize: checking size of data array to be', sum(& + write(error_unit, '(a,i3)') 'In chunked_data_scheme_timestep_final: checking size of data array to be', sum(& data_array_sizes) if (size(data_array)/=sum(data_array_sizes)) then write(errmsg, '(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(& @@ -100,12 +98,12 @@ subroutine chunked_data_scheme_timestep_finalize(data_array, errmsg, errflg) errflg = 1 return end if - end subroutine chunked_data_scheme_timestep_finalize + end subroutine chunked_data_scheme_timestep_final - !! \section arg_table_chunked_data_scheme_finalize Argument Table - !! \htmlinclude chunked_data_scheme_finalize.html + !! \section arg_table_chunked_data_scheme_final Argument Table + !! \htmlinclude chunked_data_scheme_final.html !! - subroutine chunked_data_scheme_finalize(data_array, errmsg, errflg) + subroutine chunked_data_scheme_final(data_array, errmsg, errflg) character(len=*), intent(out) :: errmsg integer, intent(out) :: errflg integer, intent(in) :: data_array(:) @@ -113,7 +111,7 @@ subroutine chunked_data_scheme_finalize(data_array, errmsg, errflg) errmsg = '' errflg = 0 ! Check size of data array - write(error_unit, '(a,i3)') 'In chunked_data_scheme_finalize: checking size of data array to be', sum(& + write(error_unit, '(a,i3)') 'In chunked_data_scheme_final: checking size of data array to be', sum(& data_array_sizes) if (size(data_array)/=sum(data_array_sizes)) then write(errmsg, '(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(& @@ -121,6 +119,6 @@ subroutine chunked_data_scheme_finalize(data_array, errmsg, errflg) errflg = 1 return end if - end subroutine chunked_data_scheme_finalize + end subroutine chunked_data_scheme_final end module chunked_data_scheme diff --git a/test_prebuild/test_chunked_data/chunked_data_scheme.meta b/end-to-end-tests/chunked_data/chunked_data_scheme.meta similarity index 91% rename from test_prebuild/test_chunked_data/chunked_data_scheme.meta rename to end-to-end-tests/chunked_data/chunked_data_scheme.meta index 13830dbf..5fb56556 100644 --- a/test_prebuild/test_chunked_data/chunked_data_scheme.meta +++ b/end-to-end-tests/chunked_data/chunked_data_scheme.meta @@ -76,24 +76,17 @@ dimensions = () type = integer intent = out -[nchunk] - standard_name = ccpp_chunk_number - long_name = number of chunk for chunked arrays in CCPP - units = index - dimensions = () - type = integer - intent = in [data_array] standard_name = chunked_data_array long_name = chunked data array units = 1 - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) type = integer intent = in ######################################################################## [ccpp-arg-table] - name = chunked_data_scheme_timestep_finalize + name = chunked_data_scheme_timestep_final type = scheme [errmsg] standard_name = ccpp_error_message @@ -120,7 +113,7 @@ ######################################################################## [ccpp-arg-table] - name = chunked_data_scheme_finalize + name = chunked_data_scheme_final type = scheme [errmsg] standard_name = ccpp_error_message diff --git a/test_prebuild/test_chunked_data/data.F90 b/end-to-end-tests/chunked_data/data.F90 similarity index 71% rename from test_prebuild/test_chunked_data/data.F90 rename to end-to-end-tests/chunked_data/data.F90 index 82c4abac..737f46c7 100644 --- a/test_prebuild/test_chunked_data/data.F90 +++ b/end-to-end-tests/chunked_data/data.F90 @@ -3,18 +3,16 @@ module data !! \section arg_table_dATa Argument Table !! \htmlinclude datA.Html !! - use ccpp_types, only: ccpp_t implicit none private - public nchunks, chunksize, chunk_begin, chunk_end, ncols - public ccpp_data_domain, ccpp_data_chunks, chunked_data_type, chunked_data_instance + public nchunks, nchunk, chunksize, chunk_begin, chunk_end, ncols + public chunked_data_type, chunked_data_instance integer, parameter :: nchunks = 4 - type(ccpp_t), target :: ccpp_data_domain - type(ccpp_t), dimension(nchunks), target :: ccpp_data_chunks + integer :: nchunk integer, parameter, dimension(nchunks) :: chunksize = (/6, 6, 6, 3/) integer, parameter, dimension(nchunks) :: chunk_begin = (/1, 7, 13, 19/) @@ -28,6 +26,7 @@ module data integer, dimension(:), allocatable :: array_data contains procedure :: create => chunked_data_create + procedure :: destroy => chunked_data_destroy end type chunked_data_type type(chunked_data_type) :: chunked_data_instance @@ -40,4 +39,9 @@ subroutine chunked_data_create(chunked_data_instance, ncol) allocate(chunked_data_instance%array_data(ncol)) end subroutine chunked_data_create + subroutine chunked_data_destroy(chunked_data_instance) + class(chunked_data_type), intent(inout) :: chunked_data_instance + deallocate(chunked_data_instance%array_data) + end subroutine chunked_data_destroy + end module data diff --git a/end-to-end-tests/chunked_data/data.meta b/end-to-end-tests/chunked_data/data.meta new file mode 100644 index 00000000..af3cb184 --- /dev/null +++ b/end-to-end-tests/chunked_data/data.meta @@ -0,0 +1,33 @@ +[ccpp-table-properties] + name = chunked_data_type + type = ddt + dependencies = +[ccpp-arg-table] + name = chunked_data_type + type = ddt +[array_data] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (ccpp_constant_one:horizontal_dimension) + type = integer + +[ccpp-table-properties] + name = data + type = host + dependencies = +[ccpp-arg-table] + name = data + type = host +[ncols] + standard_name = horizontal_dimension + long_name = horizontal dimension + units = count + dimensions = () + type = integer +[chunked_data_instance] + standard_name = chunked_data_type_instance + long_name = instance of derived data type chunked_data_type + units = DDT + dimensions = () + type = chunked_data_type diff --git a/end-to-end-tests/chunked_data/main.F90 b/end-to-end-tests/chunked_data/main.F90 new file mode 100644 index 00000000..36201afe --- /dev/null +++ b/end-to-end-tests/chunked_data/main.F90 @@ -0,0 +1,124 @@ +program test_chunked_data + + use, intrinsic :: iso_fortran_env, only: error_unit + + use data, only: nchunks, & + chunksize, & + chunk_begin, & + chunk_end, & + ncols, & + nchunk + use data, only: chunked_data_type, & + chunked_data_instance + + use test_host_ccpp_cap, only: ccpp_register, & + ccpp_init, & + ccpp_physics_init, & + ccpp_physics_timestep_init, & + ccpp_physics_run, & + ccpp_physics_timestep_final, & + ccpp_physics_final, & + ccpp_final + + implicit none + + character(len=*), parameter :: ccpp_suite = 'chunked_data_suite' + integer :: ic, ierr + integer :: lb, ub + integer :: errflg + character(len=512) :: errmsg + + call chunked_data_instance%create(ncols) + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP register step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_register(suite_name=trim(ccpp_suite), errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_register:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_init(suite_name=trim(ccpp_suite), errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_init:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_physics_init(lb=1, ub=ncols, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_init:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics timestep init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !cdata => ccpp_data_domain + call ccpp_physics_timestep_init(lb=1, ub=ncols, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics run step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do nchunk = 1, nchunks + lb=chunk_begin(nchunk) + ub=chunk_end(nchunk) + call ccpp_physics_run(lb=lb, ub=ub, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a,i3,a)') "An error occurred in ccpp_physics_run for chunk", nchunk, ":" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + end do + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics timestep finalize step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !cdata => ccpp_data_domain + call ccpp_physics_timestep_final(lb=1, ub=ncols, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_finalize:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics finalize step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !cdata => ccpp_data_domain + call ccpp_physics_final(lb=1, ub=ncols, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_finalize:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP final step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_final(suite_name=trim(ccpp_suite), errmsg=errmsg, errflg=errflg) + call chunked_data_instance%destroy() + +end program test_chunked_data diff --git a/end-to-end-tests/chunked_data/main.meta b/end-to-end-tests/chunked_data/main.meta new file mode 100644 index 00000000..b90de81e --- /dev/null +++ b/end-to-end-tests/chunked_data/main.meta @@ -0,0 +1,65 @@ +[ccpp-table-properties] + name = main + type = control + dependencies = + +[ccpp-arg-table] + name = main + type = control +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer diff --git a/test_prebuild/test_chunked_data/suite_chunked_data_suite.xml b/end-to-end-tests/chunked_data/suite_chunked_data_suite.xml similarity index 78% rename from test_prebuild/test_chunked_data/suite_chunked_data_suite.xml rename to end-to-end-tests/chunked_data/suite_chunked_data_suite.xml index 923e5fb7..32159fae 100644 --- a/test_prebuild/test_chunked_data/suite_chunked_data_suite.xml +++ b/end-to-end-tests/chunked_data/suite_chunked_data_suite.xml @@ -1,6 +1,6 @@ - + chunked_data_scheme diff --git a/end-to-end-tests/cmake/ccpp_capgen.cmake b/end-to-end-tests/cmake/ccpp_capgen.cmake new file mode 100644 index 00000000..a6b73899 --- /dev/null +++ b/end-to-end-tests/cmake/ccpp_capgen.cmake @@ -0,0 +1,235 @@ +# CMake wrapper for ccpp_validator.py +# +# SOURCE_FILES - CMake list of Fortran source files +# METADATA_FILES - CMake list of scheme metadata files. +# TYPE - Type of metadata: SCHEME or HOST. +# SCHEME metadata is validated against +# the per-phase subroutine signatures +# in SOURCE_FILES. type=scheme and +# type=ddt tables are validated; types +# control, host, suite are hard errors. +# HOST metadata: type=host and type=ddt +# tables get module-level/derived-type +# validation against SOURCE_FILES; +# type=control is silent-skipped; +# type=scheme is rejected as a hard error. +# +function(ccpp_validator) + set(optionalArgs) + set(oneValueArgs VERBOSITY TYPE) + set(multi_value_keywords SOURCE_FILES METADATA_FILES EXTRA_FLAGS) + cmake_parse_arguments(arg "${optionalArgs}" "${oneValueArgs}" "${multi_value_keywords}" ${ARGN}) + + # Error if script file not found. + set(CCPP_VALIDATOR_CMD_LIST "${CMAKE_SOURCE_DIR}/../capgen/ccpp_validator.py") + if(NOT EXISTS ${CCPP_VALIDATOR_CMD_LIST}) + message(FATAL_ERROR "function(ccpp_validator): Could not find ccpp_validator.py. Looked for ${CCPP_VALIDATOR_CMD_LIST}.") + endif() + + # Interpret parsed arguments + if(NOT DEFINED arg_SOURCE_FILES) + message(FATAL_ERROR "function(ccpp_capgen): SOURCE_FILES not set.") + endif() + list(JOIN arg_SOURCE_FILES "," SOURCE_FILES_SEPARATED) + list(APPEND CCPP_VALIDATOR_CMD_LIST "--source-files" "${SOURCE_FILES_SEPARATED}") + + if(NOT DEFINED arg_METADATA_FILES) + message(FATAL_ERROR "function(ccpp_validator): METADATA_FILES not set.") + endif() + list(JOIN arg_METADATA_FILES "," METADATA_FILES_SEPARATED) + + if(NOT DEFINED arg_TYPE) + message(FATAL_ERROR "function(ccpp_validator): TYPE must be HOST or SCHEME") + endif() + string(TOUPPER "${arg_TYPE}" _type) + if(NOT (_type MATCHES "^(HOST|SCHEME)$")) + message(FATAL_ERROR "function(ccpp_validator): TYPE must be HOST or SCHEME") + endif() + + if(_type MATCHES "^HOST$") + list(APPEND CCPP_VALIDATOR_CMD_LIST "--host-files" "${METADATA_FILES_SEPARATED}") + endif() + if(_type MATCHES "^SCHEME$") + list(APPEND CCPP_VALIDATOR_CMD_LIST "--scheme-files" "${METADATA_FILES_SEPARATED}") + endif() + + if(DEFINED arg_VERBOSITY) + string(REPEAT "--verbose " ${arg_VERBOSITY} VERBOSE_PARAMS_SEPARATED) + separate_arguments(VERBOSE_PARAMS UNIX_COMMAND "${VERBOSE_PARAMS_SEPARATED}") + list(APPEND CCPP_VALIDATOR_CMD_LIST ${VERBOSE_PARAMS}) + endif() + + if(DEFINED arg_EXTRA_FLAGS) + list(APPEND CCPP_VALIDATOR_CMD_LIST ${arg_EXTRA_FLAGS}) + endif() + + message(STATUS "Running ccpp_validator.py from ${CMAKE_CURRENT_SOURCE_DIR}") + + unset(VALIDATOR_OUT) + execute_process(COMMAND ${CCPP_VALIDATOR_CMD_LIST} + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + OUTPUT_VARIABLE VALIDATOR_OUT + ERROR_VARIABLE VALIDATOR_OUT + RESULT_VARIABLE RES + COMMAND_ECHO STDOUT) + if(RES EQUAL 0) + message(STATUS "ccpp-validator stdout: ${VALIDATOR_OUT}") + else() + message(STATUS "${CCPP_VALIDATOR_CMD_LIST} FAILED: result = ${RES}") + message(STATUS "ccpp-validator stdout: ${VALIDATOR_OUT}") + message(FATAL_ERROR "Validation of source files failed, abort.") + endif() + +endfunction() + + +# CMake wrapper for ccpp_capgen.py +# +# TRACE - ON/OFF (Default: OFF) - Add --trace flag to capgen call +# HOST_NAME - String name of host (drives _ccpp_cap.F90 filename +# and module name; required) +# OUTPUT_ROOT - String path to put generated caps +# VERBOSITY - Number of --verbose flags to pass to capgen +# HOSTFILES - CMake list of host metadata filenames +# SCHEMEFILES - CMake list of scheme metadata files +# SUITES - CMake list of suite xml files +# KIND_SPECS - Comma-separated kind mappings, e.g. "kind_phys=REAL32" or +# "kind_phys=my_mod:kind_r4,kind_dyn=REAL64". Each pair is +# forwarded as `--kind-type ` to capgen (see the +# capgen docstring for the `=[:]` +# grammar; bare ISO specs default to iso_fortran_env). +function(ccpp_capgen) + set(optionalArgs TRACE) + set(oneValueArgs HOST_NAME OUTPUT_ROOT VERBOSITY KIND_SPECS) + set(multi_value_keywords HOSTFILES SCHEMEFILES SUITES EXTRA_FLAGS) + + cmake_parse_arguments(arg "${optionalArgs}" "${oneValueArgs}" "${multi_value_keywords}" ${ARGN}) + + # Error if script file not found. + set(CCPP_CAPGEN_CMD_LIST "${CMAKE_SOURCE_DIR}/../capgen/ccpp_capgen.py") + if(NOT EXISTS ${CCPP_CAPGEN_CMD_LIST}) + message(FATAL_ERROR "function(ccpp_capgen): Could not find ccpp_capgen.py. Looked for ${CCPP_CAPGEN_CMD_LIST}.") + endif() + + # Interpret parsed arguments + if(NOT DEFINED arg_HOSTFILES) + message(FATAL_ERROR "function(ccpp_capgen): HOSTFILES not set.") + endif() + list(JOIN arg_HOSTFILES "," HOSTFILES_SEPARATED) + list(APPEND CCPP_CAPGEN_CMD_LIST "--host-files" "${HOSTFILES_SEPARATED}") + + if(NOT DEFINED arg_SCHEMEFILES) + message(FATAL_ERROR "function(ccpp_capgen): SCHEMEFILES not set.") + endif() + list(JOIN arg_SCHEMEFILES "," SCHEMEFILES_SEPARATED) + list(APPEND CCPP_CAPGEN_CMD_LIST "--scheme-files" "${SCHEMEFILES_SEPARATED}") + + if(NOT DEFINED arg_SUITES) + message(FATAL_ERROR "function(ccpp_capgen): SUITES not set.") + endif() + list(JOIN arg_SUITES "," SUITES_SEPARATED) + list(APPEND CCPP_CAPGEN_CMD_LIST "--suites" "${SUITES_SEPARATED}") + + if(NOT DEFINED arg_HOST_NAME) + message(FATAL_ERROR "function(ccpp_capgen): HOST_NAME not set.") + endif() + list(APPEND CCPP_CAPGEN_CMD_LIST "--host-name" "${arg_HOST_NAME}") + + if(NOT DEFINED arg_OUTPUT_ROOT) + message(FATAL_ERROR "function(ccpp_capgen): OUTPUT_ROOT not set.") + endif() + #file(MAKE_DIRECTORY "${arg_OUTPUT_ROOT}") + list(APPEND CCPP_CAPGEN_CMD_LIST "--output-root" "${arg_OUTPUT_ROOT}") + + if(DEFINED arg_VERBOSITY) + string(REPEAT "--verbose " ${arg_VERBOSITY} VERBOSE_PARAMS_SEPARATED) + separate_arguments(VERBOSE_PARAMS UNIX_COMMAND "${VERBOSE_PARAMS_SEPARATED}") + list(APPEND CCPP_CAPGEN_CMD_LIST ${VERBOSE_PARAMS}) + endif() + + if(DEFINED arg_KIND_SPECS) + # Accept either a comma-separated string ("kind_phys=REAL64,kind_dyn=REAL32") + # or a CMake list of pairs. Each pair becomes a separate + # `--kind-type ` argv pair so capgen's argparse sees one + # `--kind-type` per pair (the flag is `action='append'`). + string(REPLACE "," ";" KIND_SPEC_LIST "${arg_KIND_SPECS}") + foreach(pair IN LISTS KIND_SPEC_LIST) + list(APPEND CCPP_CAPGEN_CMD_LIST "--kind-type" "${pair}") + endforeach() + endif() + + if(arg_TRACE) + list(APPEND CCPP_CAPGEN_CMD_LIST "--trace") + endif() + + if(DEFINED arg_EXTRA_FLAGS) + list(APPEND CCPP_CAPGEN_CMD_LIST ${arg_EXTRA_FLAGS}) + endif() + + message(STATUS "Running ccpp_capgen.py from ${CMAKE_CURRENT_SOURCE_DIR}") + + # Unset CAPGEN_OUT to prevent incorrect output on subsequent ccpp_capgen(...) calls + unset(CAPGEN_OUT) + execute_process(COMMAND ${CCPP_CAPGEN_CMD_LIST} + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + OUTPUT_VARIABLE CAPGEN_OUT + ERROR_VARIABLE CAPGEN_OUT + RESULT_VARIABLE RES + COMMAND_ECHO STDOUT) + + message(STATUS "ccpp-capgen stdout: ${CAPGEN_OUT}") + + if(NOT RES EQUAL 0) + message(FATAL_ERROR "CCPP cap generation FAILED: result = ${RES}") + endif() + +endfunction() + + +# CMake wrapper for ccpp_datafile.py +# +# DATATABLE - Path to generated datatable.xml file +# REPORT_NAME - String report name to get list of generated files form capgen (typically --ccpp-files) +function(ccpp_datafile) + set(mandatoryArgs DATATABLE REPORT_NAME) + cmake_parse_arguments(arg "" "${mandatoryArgs}" "" ${ARGN}) + + set(CCPP_DATAFILE_CMD "${CMAKE_SOURCE_DIR}/../capgen/ccpp_datafile.py") + + if(NOT EXISTS ${CCPP_DATAFILE_CMD}) + message(FATAL_ERROR "function(ccpp_datafile): Could not find ccpp_datafile.py. Looked for ${CCPP_DATAFILE_CMD}.") + endif() + + if(NOT DEFINED arg_REPORT_NAME) + message(FATAL_ERROR "function(ccpp_datafile): REPORT_NAME not set.") + endif() + list(APPEND CCPP_DATAFILE_CMD "${arg_REPORT_NAME}") + + if(NOT DEFINED arg_DATATABLE) + message(FATAL_ERROR "function(ccpp_datafile): DATATABLE not set.") + endif() + list(APPEND CCPP_DATAFILE_CMD "${arg_DATATABLE}") + + message(STATUS "Running ccpp_datafile from ${CMAKE_CURRENT_SOURCE_DIR}") + + # Unset CCPP_FILES to prevent incorrect output on subsequent ccpp_datafile(...) calls + unset(CCPP_FILES) + execute_process(COMMAND ${CCPP_DATAFILE_CMD} + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + OUTPUT_VARIABLE CCPP_FILES + RESULT_VARIABLE RES + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_STRIP_TRAILING_WHITESPACE + COMMAND_ECHO STDOUT) + + if(NOT RES EQUAL 0) + message(FATAL_ERROR "CCPP file retrieval FAILED: result = ${RES}") + endif() + + if(CCPP_FILES) + # Convert "," separated list from python back to ";" separated list for CMake + string(REPLACE "," ";" CCPP_FILES ${CCPP_FILES}) + endif() + set(CCPP_FILES "${CCPP_FILES}" PARENT_SCOPE) + +endfunction() diff --git a/end-to-end-tests/constituents_dim/CMakeLists.txt b/end-to-end-tests/constituents_dim/CMakeLists.txt new file mode 100644 index 00000000..f60c9b2f --- /dev/null +++ b/end-to-end-tests/constituents_dim/CMakeLists.txt @@ -0,0 +1,70 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "register_consts" "const_dim_producer" "const_dim_consumer") +set(HOST_FILES "host_data" "main") +set(SUITE_FILES "constituents_dim_suite.xml") +set(HOST "test_host") +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") + +# Run ccpp_capgen +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files (incl. the constituent framework module +# and generated ccpp_host_constituents) and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +add_executable(test_constituents_dim.x + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES_FILTERED} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES} +) +target_link_libraries(test_constituents_dim.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_constituents_dim.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_constituents_dim.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_constituents_dim + COMMAND test_constituents_dim.x) diff --git a/end-to-end-tests/constituents_dim/README.md b/end-to-end-tests/constituents_dim/README.md new file mode 100644 index 00000000..b02afc7c --- /dev/null +++ b/end-to-end-tests/constituents_dim/README.md @@ -0,0 +1,47 @@ +# constituents_dim test + +Covers variables dimensioned by the framework constituent count +number_of_ccpp_constituents (which the host never declares as a scalar — the +framework owns it), plus consuming constituents WITHOUT a flag (rule b). +register_consts registers 3 dynamic constituents, so the count is 3 for the rest +of the suite. + +Count-as-dimension cases (each a distinct capgen path): + +- Case 1 — host var dimensioned by the count. surface_upward_test_constituent_flux + (horizontal_dimension, number_of_ccpp_constituents) is host-owned; the host + sizes it to the runtime count and capgen passes the whole constituent axis as + : to const_dim_producer. +- Case 2a — non-allocatable suite var, framework allocates in init_fields. + test_constituent_workspace(number_of_ccpp_constituents) is suite-owned and not + allocatable, so suite_data_init_fields allocates it via + ccpp_model_constituents_obj(i)%num_layer_vars. +- Case 2b — allocatable suite var, the scheme allocates in _run. + test_allocatable_constituent_workspace(number_of_ccpp_constituents) is + allocatable = True; init_fields skips it and const_dim_producer allocates it + using the count received as a scalar (number_of_ccpp_constituents --> + ccpp_model_constituents_obj(inst)%num_layer_vars). final_fields frees both + suite workspaces. + +Rule (b) cases — consuming a constituent without re-flagging it. const_dim_producer +flags and writes them; const_dim_consumer reads them with NO constituent flag, and +capgen infers them from the producer's flags: + +- Base constituent: qbase (test_constituent_one) is flagged advected = True on the + producer (intent=inout) and read unflagged by the consumer; both resolve to + ccpp_model_constituents_obj(inst)%vars_layer(:, index_of_test_constituent_one). +- Constituent tendency: qtend (tendency_of_test_constituent_one) is flagged + constituent = True on the producer (intent=out) and read unflagged by the + consumer; both resolve to ...%vars_layer_tend(:, index_of_test_constituent_one). + +Whether a name is a constituent is the host's decision (CAM-SIMA vs CCPP-SCM), so +the consumer never carries the flag; capgen infers it from the scheme-wide flag +set, and host/earlier-suite provision wins. + +The producer fills the workspaces and verifies Case 1; the consumer verifies +Cases 2a/2b and the rule (b) reads. Any mismatch sets errcode, failing the run. +Built with -fcheck=all, so allocation/teardown errors fail the test. + +The 2a vs 2b contrast is the same ownership rule as suite_allocate: a +non-allocatable suite var is framework-owned (allocated in init_fields); an +allocatable = True one is scheme-owned (allocated in _run). diff --git a/end-to-end-tests/constituents_dim/const_dim_consumer.F90 b/end-to-end-tests/constituents_dim/const_dim_consumer.F90 new file mode 100644 index 00000000..d96b6444 --- /dev/null +++ b/end-to-end-tests/constituents_dim/const_dim_consumer.F90 @@ -0,0 +1,68 @@ +!>\file const_dim_consumer.F90 +!! Consumes the two suite-owned workspaces produced by const_dim_producer and +!! verifies their contents. Both are dimensioned by number_of_ccpp_constituents; +!! cwork was allocated by the framework (init_fields, Case 2a) and awork by the +!! producing scheme (_run, Case 2b). Receiving them through plain (non-allocatable) +!! dummies exercises capgen passing the allocated components to a consumer. + +module const_dim_consumer + + use ccpp_kinds, only: kind_phys + + implicit none + + private + public :: const_dim_consumer_run + +contains + + !! \section arg_table_const_dim_consumer_run Argument Table + !! \htmlinclude const_dim_consumer_run.html + !! + subroutine const_dim_consumer_run(cwork, awork, qbase, qtend, errmsg, errcode) + real(kind=kind_phys), intent(in) :: cwork(:) + real(kind=kind_phys), intent(in) :: awork(:) + real(kind=kind_phys), intent(in) :: qbase(:, :) + real(kind=kind_phys), intent(in) :: qtend(:, :) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errcode + + integer :: m + + errmsg = '' + errcode = 0 + + ! Case 2a: framework-allocated suite workspace, filled by the producer. + do m = 1, size(cwork) + if (cwork(m) /= real(10 * m, kind_phys)) then + errcode = 1 + errmsg = 'Case 2a: framework-allocated suite workspace has wrong value' + return + end if + end do + + ! Case 2b: scheme-allocated suite workspace, filled by the producer. + do m = 1, size(awork) + if (awork(m) /= real(100 * m, kind_phys)) then + errcode = 1 + errmsg = 'Case 2b: scheme-allocated suite workspace has wrong value' + return + end if + end do + + ! Rule (b): qbase (base constituent) and qtend (constituent tendency) carry + ! NO constituent flag here; capgen infers them from the producer's flags and + ! reads the same framework columns (vars_layer / vars_layer_tend). + if (any(qbase /= 42.0_kind_phys)) then + errcode = 1 + errmsg = 'rule b: unflagged base-constituent consumer read the wrong value' + return + end if + if (any(qtend /= 7.0_kind_phys)) then + errcode = 1 + errmsg = 'rule b: unflagged constituent-tendency consumer read the wrong value' + return + end if + end subroutine const_dim_consumer_run + +end module const_dim_consumer diff --git a/end-to-end-tests/constituents_dim/const_dim_consumer.meta b/end-to-end-tests/constituents_dim/const_dim_consumer.meta new file mode 100644 index 00000000..40e4a09f --- /dev/null +++ b/end-to-end-tests/constituents_dim/const_dim_consumer.meta @@ -0,0 +1,55 @@ +[ccpp-table-properties] + name = const_dim_consumer + type = scheme + dependencies = + +[ccpp-arg-table] + name = const_dim_consumer_run + type = scheme +[cwork] + standard_name = test_constituent_workspace + long_name = non-allocatable suite workspace dimensioned by the count (Case 2a) + units = 1 + dimensions = (number_of_ccpp_constituents) + type = real + kind = kind_phys + intent = in +[awork] + standard_name = test_allocatable_constituent_workspace + long_name = allocatable suite workspace dimensioned by the count (Case 2b) + units = 1 + dimensions = (number_of_ccpp_constituents) + type = real + kind = kind_phys + intent = in +[qbase] + standard_name = test_constituent_one + long_name = base constituent read WITHOUT a constituent flag (rule b consumer) + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = in +[qtend] + standard_name = tendency_of_test_constituent_one + long_name = constituent tendency read WITHOUT a constituent flag (rule b consumer) + units = kg kg-1 s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = in +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errcode] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/constituents_dim/const_dim_producer.F90 b/end-to-end-tests/constituents_dim/const_dim_producer.F90 new file mode 100644 index 00000000..60a8828c --- /dev/null +++ b/end-to-end-tests/constituents_dim/const_dim_producer.F90 @@ -0,0 +1,72 @@ +!>\file const_dim_producer.F90 +!! Exercises three ways a variable can be dimensioned by the framework +!! constituent count number_of_ccpp_constituents: +!! Case 1 - consume a HOST array dimensioned by the count (passed as ':'). +!! Case 2a - fill a non-allocatable SUITE workspace the framework allocated +!! in init_fields (sized to ccpp_model_constituents_obj(i)%num_layer_vars). +!! Case 2b - allocate an allocatable SUITE workspace here in _run, sized by the +!! count received as a scalar argument. + +module const_dim_producer + + use ccpp_kinds, only: kind_phys + + implicit none + + private + public :: const_dim_producer_run + +contains + + !! \section arg_table_const_dim_producer_run Argument Table + !! \htmlinclude const_dim_producer_run.html + !! + subroutine const_dim_producer_run(coupler_flux, cwork, n_const, awork, & + qbase, qtend, errmsg, errcode) + real(kind=kind_phys), intent(in) :: coupler_flux(:, :) + real(kind=kind_phys), intent(out) :: cwork(:) + integer, intent(in) :: n_const + real(kind=kind_phys), allocatable, intent(out) :: awork(:) + real(kind=kind_phys), intent(inout) :: qbase(:, :) + real(kind=kind_phys), intent(out) :: qtend(:, :) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errcode + + integer :: m, i + + errmsg = '' + errcode = 0 + + ! Case 1: the host filled coupler_flux(i, m) = m; the constituent axis was + ! passed whole (':'), so size(coupler_flux, 2) is the constituent count. + do m = 1, size(coupler_flux, 2) + do i = 1, size(coupler_flux, 1) + if (coupler_flux(i, m) /= real(m, kind_phys)) then + errcode = 1 + errmsg = 'Case 1: coupler_flux dimensioned by ' // & + 'number_of_ccpp_constituents has the wrong value' + return + end if + end do + end do + + ! Case 2a: fill the framework-allocated (init_fields) suite workspace. + do m = 1, size(cwork) + cwork(m) = real(10 * m, kind_phys) + end do + + ! Case 2b: this scheme owns the allocation, sized by the scalar count which + ! the framework resolves to ccpp_model_constituents_obj(inst)%num_layer_vars. + allocate(awork(n_const)) + do m = 1, n_const + awork(m) = real(100 * m, kind_phys) + end do + + ! Rule (b), producer side: flag a base constituent (advected) and a + ! constituent tendency (constituent=true) and write known values into their + ! framework columns. const_dim_consumer reads both with NO flag (inference). + qbase = 42.0_kind_phys + qtend = 7.0_kind_phys + end subroutine const_dim_producer_run + +end module const_dim_producer diff --git a/end-to-end-tests/constituents_dim/const_dim_producer.meta b/end-to-end-tests/constituents_dim/const_dim_producer.meta new file mode 100644 index 00000000..777644f8 --- /dev/null +++ b/end-to-end-tests/constituents_dim/const_dim_producer.meta @@ -0,0 +1,73 @@ +[ccpp-table-properties] + name = const_dim_producer + type = scheme + dependencies = + +[ccpp-arg-table] + name = const_dim_producer_run + type = scheme +[coupler_flux] + standard_name = surface_upward_test_constituent_flux + long_name = host array dimensioned by the constituent count (Case 1) + units = kg m-2 s-1 + dimensions = (horizontal_dimension, number_of_ccpp_constituents) + type = real + kind = kind_phys + intent = in +[cwork] + standard_name = test_constituent_workspace + long_name = non-allocatable suite workspace dimensioned by the count (Case 2a) + units = 1 + dimensions = (number_of_ccpp_constituents) + type = real + kind = kind_phys + intent = out +[n_const] + standard_name = number_of_ccpp_constituents + long_name = number of CCPP constituents + units = count + dimensions = () + type = integer + intent = in +[awork] + standard_name = test_allocatable_constituent_workspace + long_name = allocatable suite workspace dimensioned by the count (Case 2b) + units = 1 + dimensions = (number_of_ccpp_constituents) + type = real + kind = kind_phys + intent = out + allocatable = True +[qbase] + standard_name = test_constituent_one + long_name = base constituent modified in place (rule b producer; advected) + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout + advected = True +[qtend] + standard_name = tendency_of_test_constituent_one + long_name = constituent tendency produced into the framework tendency array + units = kg kg-1 s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = out + constituent = True +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errcode] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/constituents_dim/constituents_dim_suite.xml b/end-to-end-tests/constituents_dim/constituents_dim_suite.xml new file mode 100644 index 00000000..819c67bf --- /dev/null +++ b/end-to-end-tests/constituents_dim/constituents_dim_suite.xml @@ -0,0 +1,9 @@ + + + + + register_consts + const_dim_producer + const_dim_consumer + + diff --git a/end-to-end-tests/constituents_dim/host_data.F90 b/end-to-end-tests/constituents_dim/host_data.F90 new file mode 100644 index 00000000..c2726819 --- /dev/null +++ b/end-to-end-tests/constituents_dim/host_data.F90 @@ -0,0 +1,23 @@ +module host_data + + !! \section arg_table_host_data Argument Table + !! \htmlinclude host_data.html + !! + use ccpp_kinds, only: kind_phys + + implicit none + + private + + public :: ncols, pver, coupler_flux + + ! Small single-chunk domain. + integer, parameter :: ncols = 4 + integer, parameter :: pver = 4 + + ! Case 1: a host-owned array dimensioned by number_of_ccpp_constituents. + ! The host allocates it to the runtime constituent count; capgen passes the + ! whole constituent axis (':') to the consuming scheme. + real(kind=kind_phys), allocatable, target :: coupler_flux(:, :) + +end module host_data diff --git a/end-to-end-tests/constituents_dim/host_data.meta b/end-to-end-tests/constituents_dim/host_data.meta new file mode 100644 index 00000000..e82be909 --- /dev/null +++ b/end-to-end-tests/constituents_dim/host_data.meta @@ -0,0 +1,26 @@ +[ccpp-table-properties] + name = host_data + type = host + dependencies = +[ccpp-arg-table] + name = host_data + type = host +[ncols] + standard_name = horizontal_dimension + units = count + type = integer + protected = True + dimensions = () +[pver] + standard_name = vertical_layer_dimension + units = count + type = integer + protected = True + dimensions = () +[coupler_flux] + standard_name = surface_upward_test_constituent_flux + long_name = host-owned array dimensioned by the framework constituent count (Case 1) + units = kg m-2 s-1 + dimensions = (horizontal_dimension, number_of_ccpp_constituents) + type = real + kind = kind_phys diff --git a/end-to-end-tests/constituents_dim/main.F90 b/end-to-end-tests/constituents_dim/main.F90 new file mode 100644 index 00000000..da2062ff --- /dev/null +++ b/end-to-end-tests/constituents_dim/main.F90 @@ -0,0 +1,123 @@ +program test_constituents_dim + + use, intrinsic :: iso_fortran_env, only: output_unit, & + error_unit + + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + use host_data, only: ncols, & + pver, & + coupler_flux + + use test_host_ccpp_cap, only: ccpp_register, & + ccpp_register_constituents, & + ccpp_number_constituents, & + ccpp_initialize_constituents, & + ccpp_init, & + ccpp_physics_init, & + ccpp_physics_timestep_init, & + ccpp_physics_run, & + ccpp_physics_timestep_final, & + ccpp_physics_final, & + ccpp_final, & + ccpp_deallocate_dynamic_constituents + + implicit none + + character(len=*), parameter :: ccpp_suite = 'constituents_dim_suite' + character(len=512) :: errmsg + integer :: errcode + integer :: num_const + integer :: m, i + type(ccpp_constituent_properties_t), allocatable, target :: host_constituents(:) + + errcode = 0 + errmsg = '' + + ! Register phase: register_consts_register registers 3 dynamic constituents, + ! which is what gives the suite a non-trivial number_of_ccpp_constituents. + call ccpp_register(suite_name=trim(ccpp_suite), errmsg=errmsg, errcode=errcode) + call check('ccpp_register') + + ! This test registers all constituents on the scheme side; the host adds none. + allocate(host_constituents(0)) + call ccpp_register_constituents(host_constituents, errmsg=errmsg, errcode=errcode) + call check('ccpp_register_constituents') + + call ccpp_number_constituents(num_const, errmsg=errmsg, errcode=errcode) + call check('ccpp_number_constituents') + if (num_const < 1) then + write(error_unit, '(a,i0)') & + 'Error: expected at least one constituent, got ', num_const + stop 1 + end if + + call ccpp_initialize_constituents(ncols=ncols, num_layers=pver, & + errcode=errcode, errmsg=errmsg) + call check('ccpp_initialize_constituents') + + ! Case 1: the host owns coupler_flux and sizes it to the constituent count. + ! capgen passes the whole constituent axis (':') to const_dim_producer_run. + allocate(coupler_flux(ncols, num_const)) + do m = 1, num_const + do i = 1, ncols + coupler_flux(i, m) = real(m, kind_phys) + end do + end do + + ! ccpp_init -> suite_data_init_fields allocates the non-allocatable suite + ! workspace (Case 2a) using ccpp_model_constituents_obj(i)%num_layer_vars. + call ccpp_init(suite_name=trim(ccpp_suite), errmsg=errmsg, errcode=errcode) + call check('ccpp_init') + + call ccpp_physics_init(suite_name=trim(ccpp_suite), group_name='all', & + col_start=1, col_end=ncols, thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) + call check('ccpp_physics_init') + + call ccpp_physics_timestep_init(suite_name=trim(ccpp_suite), group_name='all', & + col_start=1, col_end=ncols, thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) + call check('ccpp_physics_timestep_init') + + ! Producer fills/allocates the workspaces and checks Case 1; consumer verifies + ! Cases 2a/2b. Any mismatch sets errcode inside the schemes. + call ccpp_physics_run(suite_name=trim(ccpp_suite), group_name='all', & + col_start=1, col_end=ncols, thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) + call check('ccpp_physics_run') + + call ccpp_physics_timestep_final(suite_name=trim(ccpp_suite), group_name='all', & + col_start=1, col_end=ncols, thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) + call check('ccpp_physics_timestep_final') + + call ccpp_physics_final(suite_name=trim(ccpp_suite), group_name='all', & + col_start=1, col_end=ncols, thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) + call check('ccpp_physics_final') + + ! ccpp_final -> suite_data_final_fields frees both suite workspaces (guarded). + call ccpp_final(suite_name=trim(ccpp_suite), errmsg=errmsg, errcode=errcode) + call check('ccpp_final') + + call ccpp_deallocate_dynamic_constituents() + deallocate(host_constituents) + if (allocated(coupler_flux)) deallocate(coupler_flux) + + write(output_unit, '(a,i0,a)') & + 'PASS: constituents_dim (number_of_ccpp_constituents = ', num_const, ')' + +contains + + subroutine check(phase) + character(len=*), intent(in) :: phase + if (errcode /= 0) then + write(error_unit, '(a)') 'An error occurred in ' // trim(phase) // ':' + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + end subroutine check + +end program test_constituents_dim diff --git a/end-to-end-tests/constituents_dim/main.meta b/end-to-end-tests/constituents_dim/main.meta new file mode 100644 index 00000000..fee18c58 --- /dev/null +++ b/end-to-end-tests/constituents_dim/main.meta @@ -0,0 +1,65 @@ +[ccpp-table-properties] + name = main + type = control + dependencies = + +[ccpp-arg-table] + name = main + type = control +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ col_start ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = count + dimensions = () + type = integer +[ col_end ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = count + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errcode ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer diff --git a/end-to-end-tests/constituents_dim/register_consts.F90 b/end-to-end-tests/constituents_dim/register_consts.F90 new file mode 100644 index 00000000..cfcba24a --- /dev/null +++ b/end-to-end-tests/constituents_dim/register_consts.F90 @@ -0,0 +1,55 @@ +!>\file register_consts.F90 +!! Register-phase scheme that registers three dynamic constituents. Declaring a +!! ccpp_constituent_properties_t(:) argument is what activates capgen's +!! constituent machinery, giving the suite a meaningful +!! number_of_ccpp_constituents (= 3 here) for the rest of this test. + +module register_consts + + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + implicit none + + private + public :: register_consts_register + +contains + + !! \section arg_table_register_consts_register Argument Table + !! \htmlinclude register_consts_register.html + !! + subroutine register_consts_register(dyn_const, errmsg, errcode) + type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errcode + + errmsg = '' + errcode = 0 + + allocate(dyn_const(3), stat=errcode) + if (errcode /= 0) then + errmsg = 'Error allocating dyn_const in register_consts_register' + return + end if + + call dyn_const(1)%instantiate( & + std_name='test_constituent_one', long_name='test constituent one', & + diag_name='TEST_CONST_1', units='kg kg-1', default_value=0._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + errcode=errcode, errmsg=errmsg) + if (errcode /= 0) return + call dyn_const(2)%instantiate( & + std_name='test_constituent_two', long_name='test constituent two', & + diag_name='TEST_CONST_2', units='kg kg-1', default_value=0._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + errcode=errcode, errmsg=errmsg) + if (errcode /= 0) return + call dyn_const(3)%instantiate( & + std_name='test_constituent_three', long_name='test constituent three', & + diag_name='TEST_CONST_3', units='kg kg-1', default_value=0._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + errcode=errcode, errmsg=errmsg) + end subroutine register_consts_register + +end module register_consts diff --git a/test_prebuild/test_track_variables/scheme_4.meta b/end-to-end-tests/constituents_dim/register_consts.meta similarity index 52% rename from test_prebuild/test_track_variables/scheme_4.meta rename to end-to-end-tests/constituents_dim/register_consts.meta index 5464693b..4850032c 100644 --- a/test_prebuild/test_track_variables/scheme_4.meta +++ b/end-to-end-tests/constituents_dim/register_consts.meta @@ -1,28 +1,28 @@ [ccpp-table-properties] - name = scheme_4 + name = register_consts type = scheme + dependencies = -######################################################################## [ccpp-arg-table] - name = scheme_4_run + name = register_consts_register type = scheme -[ps] - standard_name = surface_air_pressure - long_name = surface pressure - units = Pa - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in +[dyn_const] + standard_name = dynamic_constituents_for_register_consts + long_name = dynamic constituents registered by this test scheme + units = none + dimensions = (:) + type = ccpp_constituent_properties_t + intent = out + allocatable = True [errmsg] standard_name = ccpp_error_message long_name = error message for error handling in CCPP units = none dimensions = () type = character - kind = len=* + kind = len=512 intent = out -[errflg] +[errcode] standard_name = ccpp_error_code long_name = error code for error handling in CCPP units = 1 diff --git a/end-to-end-tests/ddthost/CMakeLists.txt b/end-to-end-tests/ddthost/CMakeLists.txt new file mode 100644 index 00000000..d8f612b0 --- /dev/null +++ b/end-to-end-tests/ddthost/CMakeLists.txt @@ -0,0 +1,75 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "environ_conditions" "setup_coeffs" "temp_adjust" "temp_calc_adjust" "temp_set" "make_ddt") +set(HOST_FILES "host_ccpp_ddt" "test_host_data" "test_host_mod" "test_host") +set(SUITE_FILES "ddt_suite.xml" "temp_suite.xml") +set(HOST "test_host") +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") + +# Run ccpp_capgen +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +set(EXTRA_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/../utils/test_utils.F90 +) + +add_executable(test_ddthost.x + ${EXTRA_FILES} + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES_FILTERED} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES} + test_ddt_host_integration.F90 +) +target_link_libraries(test_ddthost.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_ddthost.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_ddthost.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_ddthost + COMMAND test_ddthost.x) diff --git a/test/ddthost_test/README.md b/end-to-end-tests/ddthost/README.md similarity index 100% rename from test/ddthost_test/README.md rename to end-to-end-tests/ddthost/README.md diff --git a/test/capgen_test/ddt_suite.xml b/end-to-end-tests/ddthost/ddt_suite.xml similarity index 100% rename from test/capgen_test/ddt_suite.xml rename to end-to-end-tests/ddthost/ddt_suite.xml diff --git a/test/ddthost_test/ddthost_test_reports.py b/end-to-end-tests/ddthost/ddthost_test_reports.py similarity index 100% rename from test/ddthost_test/ddthost_test_reports.py rename to end-to-end-tests/ddthost/ddthost_test_reports.py diff --git a/test/capgen_test/source_dir1/environ_conditions.F90 b/end-to-end-tests/ddthost/environ_conditions.F90 similarity index 85% rename from test/capgen_test/source_dir1/environ_conditions.F90 rename to end-to-end-tests/ddthost/environ_conditions.F90 index 2d63366e..fd2d15d7 100644 --- a/test/capgen_test/source_dir1/environ_conditions.F90 +++ b/end-to-end-tests/ddthost/environ_conditions.F90 @@ -7,7 +7,7 @@ module environ_conditions public :: environ_conditions_init public :: environ_conditions_run - public :: environ_conditions_finalize + public :: environ_conditions_final integer, parameter :: input_model_times = 3 integer, parameter :: input_model_values(input_model_times) = (/ 31, 37, 41 /) @@ -22,7 +22,7 @@ subroutine environ_conditions_run(psurf, errmsg, errflg) ! This routine currently does nothing -- should update values real(kind=kind_phys), intent(in) :: psurf(:) - character(len=512), intent(out) :: errmsg + character(len=256), intent(out) :: errmsg integer, intent(out) :: errflg errmsg = '' @@ -41,7 +41,7 @@ subroutine environ_conditions_init(nbox, o3, hno3, ntimes, model_times, & real(kind=kind_phys), intent(out) :: hno3(:) integer, intent(out) :: ntimes integer, allocatable, intent(out) :: model_times(:) - character(len=512), intent(out) :: errmsg + character(len=256), intent(out) :: errmsg integer, intent(out) :: errflg !---------------------------------------------------------------- @@ -63,14 +63,14 @@ subroutine environ_conditions_init(nbox, o3, hno3, ntimes, model_times, & end subroutine environ_conditions_init - !> \section arg_table_environ_conditions_finalize Argument Table - !! \htmlinclude arg_table_environ_conditions_finalize.html + !> \section arg_table_environ_conditions_final Argument Table + !! \htmlinclude arg_table_environ_conditions_final.html !! - subroutine environ_conditions_finalize(ntimes, model_times, errmsg, errflg) + subroutine environ_conditions_final(ntimes, model_times, errmsg, errflg) integer, intent(in) :: ntimes integer, intent(in) :: model_times(:) - character(len=512), intent(out) :: errmsg + character(len=256), intent(out) :: errmsg integer, intent(out) :: errflg ! This routine checks the size and values of model_times @@ -91,6 +91,6 @@ subroutine environ_conditions_finalize(ntimes, model_times, errmsg, errflg) errflg = 0 end if - end subroutine environ_conditions_finalize + end subroutine environ_conditions_final end module environ_conditions diff --git a/test/ddthost_test/environ_conditions.meta b/end-to-end-tests/ddthost/environ_conditions.meta similarity index 93% rename from test/ddthost_test/environ_conditions.meta rename to end-to-end-tests/ddthost/environ_conditions.meta index 894e0e92..0d425579 100644 --- a/test/ddthost_test/environ_conditions.meta +++ b/end-to-end-tests/ddthost/environ_conditions.meta @@ -6,11 +6,10 @@ type = scheme [ psurf ] standard_name = surface_air_pressure - state_variable = true type = real kind = kind_phys units = Pa - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) intent = in [ errmsg ] standard_name = ccpp_error_message @@ -18,7 +17,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=256 intent = out [ errflg ] standard_name = ccpp_error_code @@ -69,7 +68,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=256 intent = out [ errflg ] standard_name = ccpp_error_code @@ -79,7 +78,7 @@ type = integer intent = out [ccpp-arg-table] - name = environ_conditions_finalize + name = environ_conditions_final type = scheme [ ntimes ] standard_name = number_of_model_times @@ -99,7 +98,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=256 intent = out [ errflg ] standard_name = ccpp_error_code diff --git a/test/ddthost_test/host_ccpp_ddt.F90 b/end-to-end-tests/ddthost/host_ccpp_ddt.F90 similarity index 87% rename from test/ddthost_test/host_ccpp_ddt.F90 rename to end-to-end-tests/ddthost/host_ccpp_ddt.F90 index b60c81af..d9427c74 100644 --- a/test/ddthost_test/host_ccpp_ddt.F90 +++ b/end-to-end-tests/ddthost/host_ccpp_ddt.F90 @@ -9,7 +9,7 @@ module host_ccpp_ddt type, public :: ccpp_info_t integer :: col_start ! horizontal_loop_begin integer :: col_end ! horizontal_loop_end - character(len=512) :: errmsg ! ccpp_error_message + character(len=256) :: errmsg ! ccpp_error_message integer :: errflg ! ccpp_error_code end type ccpp_info_t diff --git a/test/ddthost_test/host_ccpp_ddt.meta b/end-to-end-tests/ddthost/host_ccpp_ddt.meta similarity index 97% rename from test/ddthost_test/host_ccpp_ddt.meta rename to end-to-end-tests/ddthost/host_ccpp_ddt.meta index 56dca845..4129f461 100644 --- a/test/ddthost_test/host_ccpp_ddt.meta +++ b/end-to-end-tests/ddthost/host_ccpp_ddt.meta @@ -22,7 +22,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=256 [ errflg ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP diff --git a/test/ddthost_test/make_ddt.F90 b/end-to-end-tests/ddthost/make_ddt.F90 similarity index 92% rename from test/ddthost_test/make_ddt.F90 rename to end-to-end-tests/ddthost/make_ddt.F90 index a0de4177..514827cb 100644 --- a/test/ddthost_test/make_ddt.F90 +++ b/end-to-end-tests/ddthost/make_ddt.F90 @@ -37,7 +37,7 @@ subroutine make_ddt_run(cols, cole, o3, hno3, vmr, errmsg, errflg) real(kind=kind_phys), intent(in) :: o3(:) real(kind=kind_phys), intent(in) :: hno3(:) type(vmr_type), intent(inout) :: vmr - character(len=512), intent(out) :: errmsg + character(len=256), intent(out) :: errmsg integer, intent(out) :: errflg ! Local variable integer :: nbox @@ -68,14 +68,12 @@ end subroutine make_ddt_run !> \section arg_table_make_ddt_init Argument Table !! \htmlinclude arg_table_make_ddt_init.html !! - subroutine make_ddt_init(nbox, ccpp_info, vmr, errmsg, errflg) - use host_ccpp_ddt, only: ccpp_info_t + subroutine make_ddt_init(nbox, vmr, errmsg, errflg) ! Dummy arguments integer, intent(in) :: nbox - type(ccpp_info_t), intent(in) :: ccpp_info type(vmr_type), intent(out) :: vmr - character(len=512), intent(out) :: errmsg + character(len=256), intent(out) :: errmsg integer, intent(out) :: errflg ! This routine initializes the vmr array @@ -95,7 +93,7 @@ subroutine make_ddt_timestep_final(ncols, vmr, errmsg, errflg) ! Dummy arguments integer, intent(in) :: ncols type(vmr_type), intent(in) :: vmr - character(len=512), intent(out) :: errmsg + character(len=256), intent(out) :: errmsg integer, intent(out) :: errflg ! Local variables integer :: index diff --git a/test/ddthost_test/make_ddt.meta b/end-to-end-tests/ddthost/make_ddt.meta similarity index 91% rename from test/ddthost_test/make_ddt.meta rename to end-to-end-tests/ddthost/make_ddt.meta index 4998e917..5b667a53 100644 --- a/test/ddthost_test/make_ddt.meta +++ b/end-to-end-tests/ddthost/make_ddt.meta @@ -15,6 +15,7 @@ dimensions = (horizontal_dimension, number_of_chemical_species) type = real kind = kind_phys + [ccpp-table-properties] name = make_ddt type = scheme @@ -36,14 +37,14 @@ [ O3 ] standard_name = ozone units = ppmv - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) type = real kind = kind_phys intent = in [ HNO3 ] standard_name = nitric_acid units = ppmv - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) type = real kind = kind_phys intent = in @@ -58,7 +59,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=256 intent = out [ errflg ] standard_name = ccpp_error_code @@ -67,6 +68,7 @@ dimensions = () type = integer intent = out + [ccpp-arg-table] name = make_ddt_init type = scheme @@ -76,11 +78,6 @@ units = count dimensions = () intent = in -[ ccpp_info ] - standard_name = host_standard_ccpp_type - type = ccpp_info_t - dimensions = () - intent = in [ vmr ] standard_name = volume_mixing_ratio_ddt dimensions = () @@ -92,7 +89,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=256 intent = out [ errflg ] standard_name = ccpp_error_code @@ -121,7 +118,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=256 intent = out [ errflg ] standard_name = ccpp_error_code diff --git a/test/ddthost_test/setup_coeffs.F90 b/end-to-end-tests/ddthost/setup_coeffs.F90 similarity index 92% rename from test/ddthost_test/setup_coeffs.F90 rename to end-to-end-tests/ddthost/setup_coeffs.F90 index 09c7fcc1..60963f6c 100644 --- a/test/ddthost_test/setup_coeffs.F90 +++ b/end-to-end-tests/ddthost/setup_coeffs.F90 @@ -11,7 +11,7 @@ module setup_coeffs subroutine setup_coeffs_timestep_init(coeffs, errmsg, errflg) real(kind=kind_phys), intent(inout) :: coeffs(:) - character(len=512), intent(out) :: errmsg + character(len=256), intent(out) :: errmsg integer, intent(out) :: errflg errmsg = '' diff --git a/test/ddthost_test/setup_coeffs.meta b/end-to-end-tests/ddthost/setup_coeffs.meta similarity index 97% rename from test/ddthost_test/setup_coeffs.meta rename to end-to-end-tests/ddthost/setup_coeffs.meta index 8d0fc5f4..f911df13 100644 --- a/test/ddthost_test/setup_coeffs.meta +++ b/end-to-end-tests/ddthost/setup_coeffs.meta @@ -18,7 +18,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=256 intent = out [ errflg ] standard_name = ccpp_error_code diff --git a/test/ddthost_test/temp_adjust.F90 b/end-to-end-tests/ddthost/temp_adjust.F90 similarity index 84% rename from test/ddthost_test/temp_adjust.F90 rename to end-to-end-tests/ddthost/temp_adjust.F90 index 4ef3655d..c0c4746c 100644 --- a/test/ddthost_test/temp_adjust.F90 +++ b/end-to-end-tests/ddthost/temp_adjust.F90 @@ -10,7 +10,7 @@ module temp_adjust public :: temp_adjust_init public :: temp_adjust_run - public :: temp_adjust_finalize + public :: temp_adjust_final contains @@ -28,7 +28,7 @@ subroutine temp_adjust_run(foo, timestep, temp_prev, temp_layer, qv, ps, & real(kind=kind_phys), intent(inout) :: temp_layer(:, :) real(kind=kind_phys), intent(in) :: to_promote(:, :) real(kind=kind_phys), intent(in) :: promote_pcnst(:) - character(len=512), intent(out) :: errmsg + character(len=256), intent(out) :: errmsg integer, intent(out) :: errflg real(kind=kind_phys), optional, intent(in) :: innie real(kind=kind_phys), optional, intent(out) :: outie @@ -56,7 +56,7 @@ end subroutine temp_adjust_run !! subroutine temp_adjust_init(errmsg, errflg) - character(len=512), intent(out) :: errmsg + character(len=256), intent(out) :: errmsg integer, intent(out) :: errflg ! This routine currently does nothing @@ -66,12 +66,12 @@ subroutine temp_adjust_init(errmsg, errflg) end subroutine temp_adjust_init - !> \section arg_table_temp_adjust_finalize Argument Table - !! \htmlinclude arg_table_temp_adjust_finalize.html + !> \section arg_table_temp_adjust_final Argument Table + !! \htmlinclude arg_table_temp_adjust_final.html !! - subroutine temp_adjust_finalize(errmsg, errflg) + subroutine temp_adjust_final(errmsg, errflg) - character(len=512), intent(out) :: errmsg + character(len=256), intent(out) :: errmsg integer, intent(out) :: errflg ! This routine currently does nothing @@ -79,6 +79,6 @@ subroutine temp_adjust_finalize(errmsg, errflg) errmsg = '' errflg = 0 - end subroutine temp_adjust_finalize + end subroutine temp_adjust_final end module temp_adjust diff --git a/test/ddthost_test/temp_adjust.meta b/end-to-end-tests/ddthost/temp_adjust.meta similarity index 81% rename from test/ddthost_test/temp_adjust.meta rename to end-to-end-tests/ddthost/temp_adjust.meta index a67cef8a..12880d13 100644 --- a/test/ddthost_test/temp_adjust.meta +++ b/end-to-end-tests/ddthost/temp_adjust.meta @@ -1,13 +1,13 @@ [ccpp-table-properties] name = temp_adjust type = scheme - dependencies = qux.F90 - dependencies_path = adjust + dependencies_path = + [ccpp-arg-table] name = temp_adjust_run type = scheme [ foo ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension type = integer units = count dimensions = () @@ -23,14 +23,14 @@ [ temp_prev ] standard_name = potential_temperature_at_previous_timestep units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = in [ temp_layer ] standard_name = potential_temperature units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = inout @@ -38,7 +38,7 @@ [ qv ] standard_name = water_vapor_specific_humidity units = kg kg-1 - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = inout @@ -46,16 +46,15 @@ optional = True [ ps ] standard_name = surface_air_pressure - state_variable = true type = real kind = kind_phys units = Pa - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) intent = inout [ to_promote ] standard_name = promote_this_variable_to_suite units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = in @@ -72,7 +71,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=256 intent = out [ errflg ] standard_name = ccpp_error_code @@ -90,7 +89,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=256 intent = out [ errflg ] standard_name = ccpp_error_code @@ -100,7 +99,7 @@ type = integer intent = out [ccpp-arg-table] - name = temp_adjust_finalize + name = temp_adjust_final type = scheme [ errmsg ] standard_name = ccpp_error_message @@ -108,7 +107,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=256 intent = out [ errflg ] standard_name = ccpp_error_code diff --git a/test/ddthost_test/temp_calc_adjust.F90 b/end-to-end-tests/ddthost/temp_calc_adjust.F90 similarity index 83% rename from test/ddthost_test/temp_calc_adjust.F90 rename to end-to-end-tests/ddthost/temp_calc_adjust.F90 index 4c2d7ece..40b5866e 100644 --- a/test/ddthost_test/temp_calc_adjust.F90 +++ b/end-to-end-tests/ddthost/temp_calc_adjust.F90 @@ -10,7 +10,7 @@ module temp_calc_adjust public :: temp_calc_adjust_init public :: temp_calc_adjust_run - public :: temp_calc_adjust_finalize + public :: temp_calc_adjust_final contains @@ -24,7 +24,7 @@ subroutine temp_calc_adjust_run(nbox, timestep, temp_level, temp_calc, & real(kind=kind_phys), intent(in) :: timestep real(kind=kind_phys), intent(in) :: temp_level(:, :) real(kind=kind_phys), intent(out) :: temp_calc(:, :) - character(len=512), intent(out) :: errmsg + character(len=256), intent(out) :: errmsg integer, intent(out) :: errflg !---------------------------------------------------------------- @@ -67,7 +67,7 @@ end subroutine temp_calc_adjust_run !! subroutine temp_calc_adjust_init(errmsg, errflg) - character(len=512), intent(out) :: errmsg + character(len=256), intent(out) :: errmsg integer, intent(out) :: errflg ! This routine currently does nothing @@ -77,12 +77,12 @@ subroutine temp_calc_adjust_init(errmsg, errflg) end subroutine temp_calc_adjust_init - !> \section arg_table_temp_calc_adjust_finalize Argument Table - !! \htmlinclude arg_table_temp_calc_adjust_finalize.html + !> \section arg_table_temp_calc_adjust_final Argument Table + !! \htmlinclude arg_table_temp_calc_adjust_final.html !! - subroutine temp_calc_adjust_finalize(errmsg, errflg) + subroutine temp_calc_adjust_final(errmsg, errflg) - character(len=512), intent(out) :: errmsg + character(len=256), intent(out) :: errmsg integer, intent(out) :: errflg ! This routine currently does nothing @@ -90,6 +90,6 @@ subroutine temp_calc_adjust_finalize(errmsg, errflg) errmsg = '' errflg = 0 - end subroutine temp_calc_adjust_finalize + end subroutine temp_calc_adjust_final end module temp_calc_adjust diff --git a/test/ddthost_test/temp_calc_adjust.meta b/end-to-end-tests/ddthost/temp_calc_adjust.meta similarity index 84% rename from test/ddthost_test/temp_calc_adjust.meta rename to end-to-end-tests/ddthost/temp_calc_adjust.meta index 2a5279a4..94fb9921 100644 --- a/test/ddthost_test/temp_calc_adjust.meta +++ b/end-to-end-tests/ddthost/temp_calc_adjust.meta @@ -1,13 +1,14 @@ [ccpp-table-properties] name = temp_calc_adjust type = scheme - dependencies = foo.F90, bar.F90 + dependencies = + [ccpp-arg-table] name = temp_calc_adjust_run type = scheme process = adjusting [ nbox ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension type = integer units = count dimensions = () @@ -23,14 +24,14 @@ [ temp_level ] standard_name = potential_temperature_at_interface units = K - dimensions = (ccpp_constant_one:horizontal_loop_extent, vertical_interface_dimension) + dimensions = (ccpp_constant_one:horizontal_dimension, vertical_interface_dimension) type = real kind = kind_phys intent = in [ temp_calc ] standard_name = potential_temperature_at_previous_timestep units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = out @@ -40,7 +41,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=256 intent = out [ errflg ] standard_name = ccpp_error_code @@ -58,7 +59,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=256 intent = out [ errflg ] standard_name = ccpp_error_code @@ -68,7 +69,7 @@ type = integer intent = out [ccpp-arg-table] - name = temp_calc_adjust_finalize + name = temp_calc_adjust_final type = scheme [ errmsg ] standard_name = ccpp_error_message @@ -76,7 +77,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=256 intent = out [ errflg ] standard_name = ccpp_error_code diff --git a/test/ddthost_test/temp_set.F90 b/end-to-end-tests/ddthost/temp_set.F90 similarity index 73% rename from test/ddthost_test/temp_set.F90 rename to end-to-end-tests/ddthost/temp_set.F90 index ce1c32ed..f817e34c 100644 --- a/test/ddthost_test/temp_set.F90 +++ b/end-to-end-tests/ddthost/temp_set.F90 @@ -9,9 +9,9 @@ module temp_set private public :: temp_set_init - public :: temp_set_timestep_initialize + public :: temp_set_timestep_init public :: temp_set_run - public :: temp_set_finalize + public :: temp_set_final contains @@ -31,7 +31,7 @@ subroutine temp_set_run(ncol, lev, timestep, temp_level, temp, ps, & real(kind=kind_phys), intent(inout) :: temp_level(:, :) real(kind=kind_phys), intent(out) :: to_promote(:, :) real(kind=kind_phys), intent(out) :: promote_pcnst(:) - character(len=512), intent(out) :: errmsg + character(len=256), intent(out) :: errmsg integer, intent(out) :: errflg !---------------------------------------------------------------- integer :: ilev @@ -61,12 +61,11 @@ end subroutine temp_set_run !> \section arg_table_temp_set_init Argument Table !! \htmlinclude arg_table_temp_set_init.html !! - subroutine temp_set_init(temp_inc_in, fudge, temp_inc_set, errmsg, errflg) + subroutine temp_set_init(temp_inc_in, temp_inc_set, errmsg, errflg) real(kind=kind_phys), intent(in) :: temp_inc_in - real(kind=kind_phys), intent(in) :: fudge real(kind=kind_phys), intent(out) :: temp_inc_set - character(len=512), intent(out) :: errmsg + character(len=256), intent(out) :: errmsg integer, intent(out) :: errflg temp_inc_set = temp_inc_in @@ -76,16 +75,16 @@ subroutine temp_set_init(temp_inc_in, fudge, temp_inc_set, errmsg, errflg) end subroutine temp_set_init - !> \section arg_table_temp_set_timestep_initialize Argument Table - !! \htmlinclude arg_table_temp_set_timestep_initialize.html + !> \section arg_table_temp_set_timestep_init Argument Table + !! \htmlinclude arg_table_temp_set_timestep_init.html !! - subroutine temp_set_timestep_initialize(ncol, temp_inc, temp_level, & + subroutine temp_set_timestep_init(ncol, temp_inc, temp_level, & errmsg, errflg) integer, intent(in) :: ncol real(kind=kind_phys), intent(in) :: temp_inc real(kind=kind_phys), intent(inout) :: temp_level(:, :) - character(len=512), intent(out) :: errmsg + character(len=256), intent(out) :: errmsg integer, intent(out) :: errflg errmsg = '' @@ -93,14 +92,14 @@ subroutine temp_set_timestep_initialize(ncol, temp_inc, temp_level, & temp_level = temp_level + temp_inc - end subroutine temp_set_timestep_initialize + end subroutine temp_set_timestep_init - !> \section arg_table_temp_set_finalize Argument Table - !! \htmlinclude arg_table_temp_set_finalize.html + !> \section arg_table_temp_set_final Argument Table + !! \htmlinclude arg_table_temp_set_final.html !! - subroutine temp_set_finalize(errmsg, errflg) + subroutine temp_set_final(errmsg, errflg) - character(len=512), intent(out) :: errmsg + character(len=256), intent(out) :: errmsg integer, intent(out) :: errflg ! This routine currently does nothing @@ -108,6 +107,6 @@ subroutine temp_set_finalize(errmsg, errflg) errmsg = '' errflg = 0 - end subroutine temp_set_finalize + end subroutine temp_set_final end module temp_set diff --git a/test/ddthost_test/temp_set.meta b/end-to-end-tests/ddthost/temp_set.meta similarity index 84% rename from test/ddthost_test/temp_set.meta rename to end-to-end-tests/ddthost/temp_set.meta index b6c403ce..e3375860 100644 --- a/test/ddthost_test/temp_set.meta +++ b/end-to-end-tests/ddthost/temp_set.meta @@ -6,7 +6,7 @@ type = scheme process = setter [ ncol ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension type = integer units = count dimensions = () @@ -28,29 +28,28 @@ [ temp_level ] standard_name = potential_temperature_at_interface units = K - dimensions = (ccpp_constant_one:horizontal_loop_extent, vertical_interface_dimension) + dimensions = (ccpp_constant_one:horizontal_dimension, vertical_interface_dimension) type = real kind = kind_phys intent = inout [ temp ] standard_name = potential_temperature units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = out [ ps ] standard_name = surface_air_pressure - state_variable = true type = real kind = kind_phys units = Pa - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) intent = in [ to_promote ] standard_name = promote_this_variable_to_suite units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = out @@ -67,7 +66,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=256 intent = out [ errflg ] standard_name = ccpp_error_code @@ -88,15 +87,6 @@ type = real kind = kind_phys intent = in -[ fudge ] - standard_name = random_fudge_factor - long_name = Ignore this - units = 1 - dimensions = () - type = real - kind = kind_phys - intent = in - default_value = 1.0_kind_phys [ temp_inc_set ] standard_name = test_potential_temperature_increment long_name = Per time step potential temperature increment @@ -111,7 +101,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=256 intent = out [ errflg ] standard_name = ccpp_error_code @@ -122,7 +112,7 @@ intent = out # Timestep Initialization [ccpp-arg-table] - name = temp_set_timestep_initialize + name = temp_set_timestep_init type = scheme [ ncol ] standard_name = horizontal_dimension @@ -151,7 +141,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=256 intent = out [ errflg ] standard_name = ccpp_error_code @@ -162,7 +152,7 @@ intent = out # Finalize [ccpp-arg-table] - name = temp_set_finalize + name = temp_set_final type = scheme [ errmsg ] standard_name = ccpp_error_message @@ -170,7 +160,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=256 intent = out [ errflg ] standard_name = ccpp_error_code diff --git a/test/ddthost_test/temp_suite.xml b/end-to-end-tests/ddthost/temp_suite.xml similarity index 100% rename from test/ddthost_test/temp_suite.xml rename to end-to-end-tests/ddthost/temp_suite.xml diff --git a/end-to-end-tests/ddthost/test_ddt_host_integration.F90 b/end-to-end-tests/ddthost/test_ddt_host_integration.F90 new file mode 100644 index 00000000..8e7358d2 --- /dev/null +++ b/end-to-end-tests/ddthost/test_ddt_host_integration.F90 @@ -0,0 +1,82 @@ +program test + use test_prog, only: test_host, & + suite_info, & + cm, & + cs + + implicit none + + character(len=cs), target :: test_parts1(2) = (/ 'physics1 ', & + 'physics2 ' /) + character(len=cs), target :: test_parts2(1) = (/ 'data_prep ' /) + character(len=cm), target :: test_invars1(8) = (/ & + 'potential_temperature ', & + 'potential_temperature_at_interface ', & + 'coefficients_for_interpolation ', & + 'surface_air_pressure ', & + 'water_vapor_specific_humidity ', & + 'index_of_water_vapor_specific_humidity', & + 'potential_temperature_increment ', & + 'time_step_for_physics ' /) + character(len=cm), target :: test_outvars1(7) = (/ & + 'potential_temperature ', & + 'potential_temperature_at_interface ', & + 'coefficients_for_interpolation ', & + 'surface_air_pressure ', & + 'water_vapor_specific_humidity ', & + 'ccpp_error_code ', & + 'ccpp_error_message ' /) + character(len=cm), target :: test_reqvars1(10) = (/ & + 'potential_temperature ', & + 'potential_temperature_at_interface ', & + 'coefficients_for_interpolation ', & + 'surface_air_pressure ', & + 'water_vapor_specific_humidity ', & + 'index_of_water_vapor_specific_humidity', & + 'potential_temperature_increment ', & + 'time_step_for_physics ', & + 'ccpp_error_code ', & + 'ccpp_error_message ' /) + + character(len=cm), target :: test_invars2(3) = (/ & + 'model_times ', & + 'number_of_model_times ', & + 'surface_air_pressure ' /) + + character(len=cm), target :: test_outvars2(4) = (/ & + 'ccpp_error_code ', & + 'ccpp_error_message ', & + 'model_times ', & + 'number_of_model_times ' /) + + character(len=cm), target :: test_reqvars2(5) = (/ & + 'model_times ', & + 'number_of_model_times ', & + 'surface_air_pressure ', & + 'ccpp_error_code ', & + 'ccpp_error_message ' /) + + type(suite_info) :: test_suites(2) + logical :: run_okay + + ! Setup expected test suite info + test_suites(1)%suite_name = 'temp_suite' + test_suites(1)%suite_parts => test_parts1 + test_suites(1)%suite_input_vars => test_invars1 + test_suites(1)%suite_output_vars => test_outvars1 + test_suites(1)%suite_required_vars => test_reqvars1 + test_suites(2)%suite_name = 'ddt_suite' + test_suites(2)%suite_parts => test_parts2 + test_suites(2)%suite_input_vars => test_invars2 + test_suites(2)%suite_output_vars => test_outvars2 + test_suites(2)%suite_required_vars => test_reqvars2 + + call test_host(run_okay, test_suites) + + if (run_okay) then + stop 0 + else + stop -1 + end if + +end program test diff --git a/test/ddthost_test/test_host.F90 b/end-to-end-tests/ddthost/test_host.F90 similarity index 60% rename from test/ddthost_test/test_host.F90 rename to end-to-end-tests/ddthost/test_host.F90 index ebe175d9..9b9b6204 100644 --- a/test/ddthost_test/test_host.F90 +++ b/end-to-end-tests/ddthost/test_host.F90 @@ -9,7 +9,7 @@ module test_prog ! Public data and interfaces integer, public, parameter :: cs = 16 - integer, public, parameter :: cm = 36 + integer, public, parameter :: cm = 64 !> \section arg_table_suite_info Argument Table !! \htmlinclude arg_table_suite_info.html @@ -35,7 +35,7 @@ logical function check_suite(test_suite) integer :: sind logical :: check integer :: errflg - character(len=512) :: errmsg + character(len=256) :: errmsg character(len=128), allocatable :: test_list(:) check_suite = .true. @@ -103,14 +103,16 @@ end function check_suite !! subroutine test_host(retval, test_suites) - use host_ccpp_ddt, only: ccpp_info_t use test_host_mod, only: ncols, & num_time_steps - use test_host_ccpp_cap, only: test_host_ccpp_physics_initialize - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_initial - use test_host_ccpp_cap, only: test_host_ccpp_physics_run - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_final - use test_host_ccpp_cap, only: test_host_ccpp_physics_finalize + use test_host_ccpp_cap, only: ccpp_register + use test_host_ccpp_cap, only: ccpp_init + use test_host_ccpp_cap, only: ccpp_physics_init + use test_host_ccpp_cap, only: ccpp_physics_timestep_init + use test_host_ccpp_cap, only: ccpp_physics_run + use test_host_ccpp_cap, only: ccpp_physics_timestep_final + use test_host_ccpp_cap, only: ccpp_physics_final + use test_host_ccpp_cap, only: ccpp_final use test_host_ccpp_cap, only: ccpp_physics_suite_list use test_host_mod, only: init_data, & compare_data, & @@ -121,12 +123,14 @@ subroutine test_host(retval, test_suites) logical, intent(out) :: retval logical :: check - integer :: col_start + integer :: col_start, col_end integer :: index, sind integer :: time_step integer :: num_suites character(len=128), allocatable :: suite_names(:) - type(ccpp_info_t) :: ccpp_info + integer :: errflg + character(len=256) :: errmsg + ! Initialize our 'data' call init_data() @@ -158,61 +162,91 @@ subroutine test_host(retval, test_suites) return end if + ! Register CCPP + do sind = 1, num_suites + call ccpp_register( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in register of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + end if + end do + + ! Initialize CCPP + do sind = 1, num_suites + call ccpp_init( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in init of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + end if + end do + ! Use the suite information to setup the run do sind = 1, num_suites - call test_host_ccpp_physics_initialize(test_suites(sind)%suite_name, & - ccpp_info) - if (ccpp_info%errflg /= 0) then - write(6, '(4a)') 'ERROR in initialize of ', & - trim(test_suites(sind)%suite_name), ': ', trim(ccpp_info%errmsg) + call ccpp_physics_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='', errmsg=errmsg, errflg=errflg, & + col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in physics_init of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) end if end do + ! Loop over time steps do time_step = 1, num_time_steps ! Initialize the timestep do sind = 1, num_suites - if (ccpp_info%errflg /= 0) then + if (errflg /= 0) then exit end if - if (ccpp_info%errflg == 0) then - call test_host_ccpp_physics_timestep_initial( & - test_suites(sind)%suite_name, ccpp_info) + if (errflg == 0) then + call ccpp_physics_timestep_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='', errmsg=errmsg, errflg=errflg, & + col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1) end if - if (ccpp_info%errflg /= 0) then + if (errflg /= 0) then write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & - trim(ccpp_info%errmsg) + trim(errmsg) exit end if - if (ccpp_info%errflg /= 0) then + if (errflg /= 0) then exit end if end do do col_start = 1, ncols, 5 - if (ccpp_info%errflg /= 0) then + if (errflg /= 0) then exit end if - ccpp_info%col_start = col_start - ccpp_info%col_end = min(col_start + 4, ncols) + col_end = min(col_start + 4, ncols) do sind = 1, num_suites - if (ccpp_info%errflg /= 0) then + if (errflg /= 0) then exit end if do index = 1, size(test_suites(sind)%suite_parts) - if (ccpp_info%errflg /= 0) then + if (errflg /= 0) then exit end if - if (ccpp_info%errflg == 0) then - call test_host_ccpp_physics_run( & - test_suites(sind)%suite_name, & - test_suites(sind)%suite_parts(index), & - ccpp_info) + if (errflg == 0) then + call ccpp_physics_run( & + suite_name=test_suites(sind)%suite_name, & + group_name=test_suites(sind)%suite_parts(index), & + errmsg=errmsg, errflg=errflg, & + col_start=col_start, col_end=col_end, & + thread_num=1, nthreads=1, nphys_threads=1) end if - if (ccpp_info%errflg /= 0) then + if (errflg /= 0) then write(6, '(5a)') trim(test_suites(sind)%suite_name), & '/', trim(test_suites(sind)%suite_parts(index)), & - ': ', trim(ccpp_info%errmsg) + ': ', trim(errmsg) exit end if end do @@ -220,53 +254,79 @@ subroutine test_host(retval, test_suites) end do do sind = 1, num_suites - if (ccpp_info%errflg /= 0) then + if (errflg /= 0) then exit end if - if (ccpp_info%errflg == 0) then - call test_host_ccpp_physics_timestep_final( & - test_suites(sind)%suite_name, ccpp_info) + if (errflg == 0) then + call ccpp_physics_timestep_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='', errmsg=errmsg, errflg=errflg, & + col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1) end if - if (ccpp_info%errflg /= 0) then + if (errflg /= 0) then write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & - trim(ccpp_info%errmsg) + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_physics_timestep_final, ', & + 'Exiting...' exit end if end do end do ! End time step loop do sind = 1, num_suites - if (ccpp_info%errflg /= 0) then + if (errflg /= 0) then exit end if - if (ccpp_info%errflg == 0) then - call test_host_ccpp_physics_finalize( & - test_suites(sind)%suite_name, ccpp_info) + if (errflg == 0) then + call ccpp_physics_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='', errmsg=errmsg, errflg=errflg, & + col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1) end if - if (ccpp_info%errflg /= 0) then + if (errflg /= 0) then write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & - trim(ccpp_info%errmsg) - write(6, '(2a)') 'An error occurred in ccpp_timestep_final, ', & + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_physics_final, ', & 'Exiting...' exit end if end do - if (ccpp_info%errflg == 0) then + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_final( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + end if + if (errflg /= 0) then + write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_final, ', & + 'Exiting...' + exit + end if + end do + + if (errflg == 0) then ! Run finished without error, check answers if (.not. check_model_times()) then write(6, *) 'Model times error!' - ccpp_info%errflg = -1 + errflg = -1 else if (compare_data()) then write(6, *) 'Answers are correct!' - ccpp_info%errflg = 0 + errflg = 0 else write(6, *) 'Answers are not correct!' - ccpp_info%errflg = -1 + errflg = -1 end if end if - retval = ccpp_info%errflg == 0 + retval = errflg == 0 end subroutine test_host diff --git a/end-to-end-tests/ddthost/test_host.meta b/end-to-end-tests/ddthost/test_host.meta new file mode 100644 index 00000000..13da7afc --- /dev/null +++ b/end-to-end-tests/ddthost/test_host.meta @@ -0,0 +1,64 @@ +[ccpp-table-properties] + name = test_host + type = control + +[ccpp-arg-table] + name = test_host + type = control +[ col_start ] + standard_name = horizontal_loop_begin + type = integer + units = count + dimensions = () + protected = True +[ col_end ] + standard_name = horizontal_loop_end + type = integer + units = count + dimensions = () + protected = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer diff --git a/test/ddthost_test/test_host_data.F90 b/end-to-end-tests/ddthost/test_host_data.F90 similarity index 100% rename from test/ddthost_test/test_host_data.F90 rename to end-to-end-tests/ddthost/test_host_data.F90 diff --git a/test/ddthost_test/test_host_data.meta b/end-to-end-tests/ddthost/test_host_data.meta similarity index 89% rename from test/ddthost_test/test_host_data.meta rename to end-to-end-tests/ddthost/test_host_data.meta index df4b92b4..f195605e 100644 --- a/test/ddthost_test/test_host_data.meta +++ b/end-to-end-tests/ddthost/test_host_data.meta @@ -6,7 +6,6 @@ type = ddt [ ps ] standard_name = surface_air_pressure - state_variable = true type = real kind = kind_phys units = Pa @@ -14,7 +13,6 @@ [ u ] standard_name = eastward_wind long_name = Zonal wind - state_variable = true type = real kind = kind_phys units = m s-1 @@ -22,7 +20,6 @@ [ v ] standard_name = northward_wind long_name = Meridional wind - state_variable = true type = real kind = kind_phys units = m s-1 @@ -30,21 +27,18 @@ [ pmid ] standard_name = air_pressure long_name = Midpoint air pressure - state_variable = true type = real kind = kind_phys units = Pa dimensions = (horizontal_dimension, vertical_layer_dimension) [ q ] standard_name = constituent_mixing_ratio - state_variable = true type = real kind = kind_phys units = kg kg-1 moist or dry air depending on type dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_tracers) [ q(:,:,index_of_water_vapor_specific_humidity) ] standard_name = water_vapor_specific_humidity - state_variable = true type = real kind = kind_phys units = kg kg-1 diff --git a/test/ddthost_test/test_host_mod.F90 b/end-to-end-tests/ddthost/test_host_mod.F90 similarity index 98% rename from test/ddthost_test/test_host_mod.F90 rename to end-to-end-tests/ddthost/test_host_mod.F90 index 02eb4991..066d8a7d 100644 --- a/test/ddthost_test/test_host_mod.F90 +++ b/end-to-end-tests/ddthost/test_host_mod.F90 @@ -24,7 +24,7 @@ module test_host_mod diag2 real(kind=kind_phys) :: dt real(kind=kind_phys), parameter :: temp_inc = 0.05_kind_phys - type(physics_state) :: phys_state + type(physics_state), target :: phys_state integer :: num_model_times = -1 integer, allocatable :: model_times(:) diff --git a/test/ddthost_test/test_host_mod.meta b/end-to-end-tests/ddthost/test_host_mod.meta similarity index 98% rename from test/ddthost_test/test_host_mod.meta rename to end-to-end-tests/ddthost/test_host_mod.meta index a450ee67..e278742a 100644 --- a/test/ddthost_test/test_host_mod.meta +++ b/end-to-end-tests/ddthost/test_host_mod.meta @@ -1,9 +1,9 @@ [ccpp-table-properties] name = test_host_mod - type = module + type = host [ccpp-arg-table] name = test_host_mod - type = module + type = host [ index_qv ] standard_name = index_of_water_vapor_specific_humidity units = index diff --git a/end-to-end-tests/instances/CMakeLists.txt b/end-to-end-tests/instances/CMakeLists.txt new file mode 100644 index 00000000..c4e6fbbd --- /dev/null +++ b/end-to-end-tests/instances/CMakeLists.txt @@ -0,0 +1,64 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "unit_conv_scheme_1" "unit_conv_scheme_2") +set(HOST_FILES "data" "main") +set(SUITE_FILES "suite_unit_conv_suite.xml") +set(HOST "test_host") +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") + +# Run ccpp_capgen +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +add_executable(test_instances.x ${SCHEME_FORTRAN_FILES_FILTERED} ${HOST_FORTRAN_FILES} ${CAPGEN_FILES}) +target_link_libraries(test_instances.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_instances.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_instances.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_instances + COMMAND test_instances.x) diff --git a/test_prebuild/test_unit_conv/README.md b/end-to-end-tests/instances/README.md similarity index 97% rename from test_prebuild/test_unit_conv/README.md rename to end-to-end-tests/instances/README.md index 17ca1c58..787d0926 100644 --- a/test_prebuild/test_unit_conv/README.md +++ b/end-to-end-tests/instances/README.md @@ -4,7 +4,7 @@ 2. Run the following commands: ``` cd test_prebuild/test_unit_conv/ -rm -fr build +#rm -fr build mkdir build ../../scripts/ccpp_prebuild.py --config=ccpp_prebuild_config.py --builddir=build cd build diff --git a/end-to-end-tests/instances/data.F90 b/end-to-end-tests/instances/data.F90 new file mode 100644 index 00000000..924a209b --- /dev/null +++ b/end-to-end-tests/instances/data.F90 @@ -0,0 +1,30 @@ +module data + + !! \section arg_table_data Argument Table + !! \htmlinclude data.html + !! + use ccpp_kinds, only : kind_phys + + implicit none + + private + + public ncols, nspecies, ninstances + public instance_type, instance_data + + integer, parameter :: ncols = 4 + integer, parameter :: nspecies = 2 + integer, parameter :: ninstances = 2 + + !! \section arg_table_instance_type Argument Table + !! \htmlinclude instance_type.html + !! + type instance_type + real(kind=kind_phys), dimension(1:ncols, 1:nspecies) :: data_array + real(kind=kind_phys), dimension(1:ncols) :: data_array2 + logical :: opt_array_flag + end type instance_type + + type(instance_type), dimension(1:ninstances), target :: instance_data + +end module data diff --git a/test_prebuild/test_unit_conv/data.meta b/end-to-end-tests/instances/data.meta similarity index 78% rename from test_prebuild/test_unit_conv/data.meta rename to end-to-end-tests/instances/data.meta index 9c9ea5e7..1fccc944 100644 --- a/test_prebuild/test_unit_conv/data.meta +++ b/end-to-end-tests/instances/data.meta @@ -1,34 +1,11 @@ [ccpp-table-properties] - name = data - type = module + name = instance_type + type = ddt dependencies = + [ccpp-arg-table] - name = data - type = module -[cdata] - standard_name = ccpp_t_instance - long_name = instance of derived data type ccpp_t - units = DDT - dimensions = () - type = ccpp_t -[ncols] - standard_name = horizontal_dimension - long_name = horizontal dimension - units = count - dimensions = () - type = integer -[ncolsrun] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent - units = count - dimensions = () - type = integer -[nspecies] - standard_name = number_of_species - long_name = number of species in data array - units = count - dimensions = () - type = integer + name = instance_type + type = ddt [data_array] standard_name = data_array_all_species long_name = data array in module @@ -64,3 +41,31 @@ type = real kind = kind_phys active = (flag_for_opt_array) + + +[ccpp-table-properties] + name = data + type = host + dependencies = + +[ccpp-arg-table] + name = data + type = host +[ncols] + standard_name = horizontal_dimension + long_name = horizontal dimension + units = count + dimensions = () + type = integer +[nspecies] + standard_name = number_of_species + long_name = number of species in data array + units = count + dimensions = () + type = integer +[instance_data] + standard_name = instance_data + long_name = instance data for multi-instance test + units = ddt + dimensions = (number_of_instances) + type = instance_type diff --git a/end-to-end-tests/instances/main.F90 b/end-to-end-tests/instances/main.F90 new file mode 100644 index 00000000..2d34bb51 --- /dev/null +++ b/end-to-end-tests/instances/main.F90 @@ -0,0 +1,184 @@ +program test_unit_conv + + use, intrinsic :: iso_fortran_env, only: error_unit +#ifdef _OPENMP + use omp_lib +#endif + + use data, only: ncols, & + nspecies, ninstances + use data, only: instance_data + + use test_host_ccpp_cap, only: ccpp_register, & + ccpp_init, & + ccpp_physics_init, & + ccpp_physics_timestep_init, & + ccpp_physics_run, & + ccpp_physics_timestep_final, & + ccpp_physics_final, & + ccpp_final + + implicit none + + character(len=*), parameter :: ccpp_suite = 'unit_conv_suite' + ! An updated ccpp_validator.py should detect this - metadata has len=512 + character(len=256) :: errmsg + integer :: errflg + integer :: nphys_threads + integer :: ins + + !data_array = 1.0_8 + !data_array2 = 42.0_8 + !opt_array_flag = .true. + + instance_data(1)%data_array = -1.0_8 + instance_data(1)%data_array2 = -42.0_8 + instance_data(1)%opt_array_flag = .true. + + instance_data(2)%data_array = +1.0_8 + instance_data(2)%data_array2 = +42.0_8 + instance_data(2)%opt_array_flag = .false. + + ! Use OpenMP threading in physics (internally) +#ifdef _OPENMP + nphys_threads = omp_get_max_threads() +#else + nphys_threads = 1 +#endif + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP register step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do ins=1,ninstances + call ccpp_register(suite_name=ccpp_suite, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_register:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + end do + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do ins=1,ninstances + call ccpp_init(suite_name=ccpp_suite, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_init:" + write(error_unit, '(a)') trim(errmsg) + write(error_unit, '(a,i0)') "instance: ", ins + stop 1 + end if + end do + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do ins=1,ninstances + call ccpp_physics_init( & + suite_name=ccpp_suite, group_name='all', & + instance=ins, ninstances=ninstances, & + thread_num=1, nthreads=1, nphys_threads=nphys_threads, & + lb=1, ub=ncols, errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_init:" + write(error_unit, '(a)') trim(errmsg) + write(error_unit, '(a,i0)') "instance: ", ins + stop 1 + end if + end do + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics timestep init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do ins=1,ninstances + call ccpp_physics_timestep_init( & + suite_name=ccpp_suite, group_name='all', & + instance=ins, ninstances=ninstances, & + thread_num=1, nthreads=1, nphys_threads=nphys_threads, & + lb=1, ub=ncols, errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" + write(error_unit, '(a)') trim(errmsg) + write(error_unit, '(a,i0)') "instance: ", ins + stop 1 + end if + end do + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics run step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do ins=1,ninstances + call ccpp_physics_run( & + suite_name=ccpp_suite, group_name='all', & + instance=ins, ninstances=ninstances, & + thread_num=1, nthreads=1, nphys_threads=nphys_threads, & + lb=1, ub=ncols, errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_run:" + write(error_unit, '(a)') trim(errmsg) + write(error_unit, '(a,i0)') "instance: ", ins + stop 1 + end if + end do + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics timestep final step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do ins=1,ninstances + call ccpp_physics_timestep_final( & + suite_name=ccpp_suite, group_name='all', & + instance=ins, ninstances=ninstances, & + thread_num=1, nthreads=1, nphys_threads=nphys_threads, & + lb=1, ub=ncols, errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_final:" + write(error_unit, '(a)') trim(errmsg) + write(error_unit, '(a,i0)') "instance: ", ins + stop 1 + end if + end do + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics final step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do ins=1,ninstances + call ccpp_physics_final( & + suite_name=ccpp_suite, group_name='all', & + instance=ins, ninstances=ninstances, & + thread_num=1, nthreads=1, nphys_threads=nphys_threads, & + lb=1, ub=ncols, errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_final:" + write(error_unit, '(a)') trim(errmsg) + write(error_unit, '(a,i0)') "instance: ", ins + stop 1 + end if + end do + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP final step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do ins=1,ninstances + call ccpp_final(suite_name=ccpp_suite, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_final:" + write(error_unit, '(a)') trim(errmsg) + write(error_unit, '(a,i0)') "instance: ", ins + stop 1 + end if + end do + +end program test_unit_conv diff --git a/end-to-end-tests/instances/main.meta b/end-to-end-tests/instances/main.meta new file mode 100644 index 00000000..e1f64d13 --- /dev/null +++ b/end-to-end-tests/instances/main.meta @@ -0,0 +1,77 @@ +[ccpp-table-properties] + name = main + type = control + dependencies = + +[ccpp-arg-table] + name = main + type = control +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer +[ instance ] + standard_name = instance_number + long_name = current model instance number + units = index + dimensions = () + type = integer +[ninstances] + standard_name = number_of_instances + long_name = number of instances for multi-instance test + units = count + dimensions = () + type = integer diff --git a/test_prebuild/test_unit_conv/suite_unit_conv_suite.xml b/end-to-end-tests/instances/suite_unit_conv_suite.xml similarity index 85% rename from test_prebuild/test_unit_conv/suite_unit_conv_suite.xml rename to end-to-end-tests/instances/suite_unit_conv_suite.xml index 68d90109..0f499a42 100644 --- a/test_prebuild/test_unit_conv/suite_unit_conv_suite.xml +++ b/end-to-end-tests/instances/suite_unit_conv_suite.xml @@ -1,6 +1,6 @@ - + unit_conv_scheme_1 diff --git a/end-to-end-tests/instances/unit_conv_scheme_1.F90 b/end-to-end-tests/instances/unit_conv_scheme_1.F90 new file mode 100644 index 00000000..3cf05bf8 --- /dev/null +++ b/end-to-end-tests/instances/unit_conv_scheme_1.F90 @@ -0,0 +1,86 @@ +!>\file unit_conv_scheme_1.F90 +!! This file contains a unit_conv_scheme_1 CCPP scheme that does nothing +!! except requesting the minimum, mandatory variables. + +module unit_conv_scheme_1 + + use, intrinsic :: iso_fortran_env, only: error_unit + use ccpp_kinds, only : kind_phys + implicit none + + private + public :: unit_conv_scheme_1_run + + ! This is for unit testing only + real(kind=kind_phys), parameter, dimension(1:2) :: target_values = (/-1.0_kind_phys, 1.0_kind_phys/) + real(kind=kind_phys), parameter, dimension(1:2) :: target_values2 = (/-42.0_kind_phys, 42.0_kind_phys/) + +contains + + !! \section arg_table_unit_conv_scheme_1_run Argument Table + !! \htmlinclude unit_conv_scheme_1_run.html + !! + subroutine unit_conv_scheme_1_run(instance, data_array, data_array2, data_array_opt, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: instance + real(kind=kind_phys), intent(inout) :: data_array(:) + real(kind=kind_phys), intent(inout) :: data_array2(:) + real(kind=kind_phys), intent(inout), optional :: data_array_opt(:) + + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + + ! Check values in data array + write(error_unit, '(a,e12.4)') & + 'In unit_conv_scheme_1_run: checking min/max values of data array to be approximately ', & + target_values(instance) + if (abs(minval(data_array) - target_values(instance)) > 0.01_kind_phys .or. & + abs(maxval(data_array) - target_values(instance)) > 0.01_kind_phys) then + write(errmsg, '(3(a,e12.4),a)') & + "Error in unit_conv_scheme_1_run, expected values for data_array of approximately ", & + target_values(instance), " but got [ ", minval(data_array), " : ", maxval(data_array), " ]" + errflg = 1 + return + end if + ! Check values in data array2 + write(error_unit, '(a,e12.4)') & + 'In unit_conv_scheme_1_run: checking min/max values of data array 2 to be approximately ', & + target_values2(instance) + if (abs(minval(data_array2) - target_values2(instance)) > 0.01_kind_phys .or. & + abs(maxval(data_array2) - target_values2(instance)) > 0.01_kind_phys) then + write(errmsg, '(3(a,e12.4),a)') & + "Error in unit_conv_scheme_1_run, expected values for data array 2 of approximately ", & + target_values2(instance), " but got [ ", minval(data_array2), " : ", maxval(data_array2), " ]" + errflg = 1 + return + end if + ! Check for presence of optional data array, then check its values + write(error_unit, '(a)') 'In unit_conv_scheme_1_run: checking for presence of optional data array' + if (instance==1) then + if (.not. present(data_array_opt)) then + write(errmsg, '(a)') 'Error in unit_conv_scheme_1_run, optional data array expected but not present' + write(errmsg, '(a,i0)') 'for instance ', instance + errflg = 1 + return + end if + write(error_unit, '(a,e12.4)') & + 'In unit_conv_scheme_1_run: checking min/max values of optional data array to be approximately ', target_values(instance) + if (abs(minval(data_array_opt) - target_values(instance)) > 0.01_kind_phys .or. & + abs(maxval(data_array_opt) - target_values(instance)) > 0.01_kind_phys) then + write(errmsg, '(3(a,e12.4),a)') 'Error in unit_conv_scheme_1_run, expected values of approximately ', & + target_values(instance), ' but got [ ', minval(data_array_opt), ' : ', maxval(data_array_opt), ' ]' + errflg = 1 + return + end if + else if (instance==2 .and. present(data_array_opt)) then + write(errmsg, '(a)') 'Error in unit_conv_scheme_1_run, optional data array not expected but present' + write(errmsg, '(a,i0)') 'for instance ', instance + errflg = 1 + return + end if + + end subroutine unit_conv_scheme_1_run + +end module unit_conv_scheme_1 diff --git a/test_prebuild/test_unit_conv/unit_conv_scheme_1.meta b/end-to-end-tests/instances/unit_conv_scheme_1.meta similarity index 77% rename from test_prebuild/test_unit_conv/unit_conv_scheme_1.meta rename to end-to-end-tests/instances/unit_conv_scheme_1.meta index befb19bd..ef096774 100644 --- a/test_prebuild/test_unit_conv/unit_conv_scheme_1.meta +++ b/end-to-end-tests/instances/unit_conv_scheme_1.meta @@ -22,11 +22,18 @@ dimensions = () type = integer intent = out +[instance] + standard_name = instance_number + long_name = instance number for testing multi-instance support + units = index + dimensions = () + type = integer + intent = in [data_array] standard_name = data_array long_name = data array in m units = m - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) type = real kind = kind_phys intent = inout @@ -34,7 +41,7 @@ standard_name = data_array2 long_name = data array in J kg-1 units = J kg-1 - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) type = real kind = kind_phys intent = inout @@ -42,7 +49,7 @@ standard_name = data_array_opt long_name = optional data array in m units = m - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) type = real kind = kind_phys intent = inout diff --git a/end-to-end-tests/instances/unit_conv_scheme_2.F90 b/end-to-end-tests/instances/unit_conv_scheme_2.F90 new file mode 100644 index 00000000..f1d8fbed --- /dev/null +++ b/end-to-end-tests/instances/unit_conv_scheme_2.F90 @@ -0,0 +1,86 @@ +!>\file unit_conv_scheme_2.F90 +!! This file contains a unit_conv_scheme_2 CCPP scheme that does nothing +!! except requesting the minimum, mandatory variables. + +module unit_conv_scheme_2 + + use, intrinsic :: iso_fortran_env, only: error_unit + use ccpp_kinds, only : kind_phys + implicit none + + private + public :: unit_conv_scheme_2_run + + ! This is for unit testing only + real(kind=kind_phys), parameter, dimension(1:2) :: target_values = (/-1.0E-3_kind_phys, 1.0E-3_kind_phys/) + real(kind=kind_phys), parameter, dimension(1:2) :: target_values2 = (/-42.0_kind_phys, 42.0_kind_phys/) + +contains + + !! \section arg_table_unit_conv_scheme_2_run Argument Table + !! \htmlinclude unit_conv_scheme_2_run.html + !! + subroutine unit_conv_scheme_2_run(instance, data_array, data_array2, data_array_opt, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: instance + real(kind=kind_phys), intent(inout) :: data_array(:) + real(kind=kind_phys), intent(inout) :: data_array2(:) + real(kind=kind_phys), intent(inout), optional :: data_array_opt(:) + + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + + ! Check values in data array + write(error_unit, '(a,e12.4)') & + 'In unit_conv_scheme_2_run: checking min/max values of data array to be approximately ', & + target_values(instance) + if (abs(minval(data_array) - target_values(instance)) > 0.01_kind_phys .or. & + abs(maxval(data_array) - target_values(instance)) > 0.01_kind_phys) then + write(errmsg, '(3(a,e12.4),a)') & + "Error in unit_conv_scheme_2_run, expected values for data_array of approximately ", & + target_values(instance), " but got [ ", minval(data_array), " : ", maxval(data_array), " ]" + errflg = 1 + return + end if + ! Check values in data array2 + write(error_unit, '(a,e12.4)') & + 'In unit_conv_scheme_2_run: checking min/max values of data array 2 to be approximately ', & + target_values2(instance) + if (abs(minval(data_array2) - target_values2(instance)) > 0.01_kind_phys .or. & + abs(maxval(data_array2) - target_values2(instance)) > 0.01_kind_phys) then + write(errmsg, '(3(a,e12.4),a)') & + "Error in unit_conv_scheme_2_run, expected values for data array 2 of approximately ", & + target_values2(instance), " but got [ ", minval(data_array2), " : ", maxval(data_array2), " ]" + errflg = 1 + return + end if + ! Check for presence of optional data array, then check its values + write(error_unit, '(a)') 'In unit_conv_scheme_2_run: checking for presence of optional data array' + if (instance==1) then + if (.not. present(data_array_opt)) then + write(errmsg, '(a)') 'Error in unit_conv_scheme_2_run, optional data array expected but not present' + write(errmsg, '(a,i0)') 'for instance ', instance + errflg = 1 + return + end if + write(error_unit, '(a,e12.4)') & + 'In unit_conv_scheme_2_run: checking min/max values of optional data array to be approximately ', target_values(instance) + if (abs(minval(data_array_opt) - target_values(instance)) > 0.01_kind_phys .or. & + abs(maxval(data_array_opt) - target_values(instance)) > 0.01_kind_phys) then + write(errmsg, '(3(a,e12.4),a)') 'Error in unit_conv_scheme_2_run, expected values of approximately ', & + target_values(instance), ' but got [ ', minval(data_array_opt), ' : ', maxval(data_array_opt), ' ]' + errflg = 1 + return + end if + else if (instance==2 .and. present(data_array_opt)) then + write(errmsg, '(a)') 'Error in unit_conv_scheme_2_run, optional data array not expected but present' + write(errmsg, '(a,i0)') 'for instance ', instance + errflg = 1 + return + end if + + end subroutine unit_conv_scheme_2_run + +end module unit_conv_scheme_2 diff --git a/test_prebuild/test_unit_conv/unit_conv_scheme_2.meta b/end-to-end-tests/instances/unit_conv_scheme_2.meta similarity index 77% rename from test_prebuild/test_unit_conv/unit_conv_scheme_2.meta rename to end-to-end-tests/instances/unit_conv_scheme_2.meta index 68e4b063..e1b916c2 100644 --- a/test_prebuild/test_unit_conv/unit_conv_scheme_2.meta +++ b/end-to-end-tests/instances/unit_conv_scheme_2.meta @@ -22,11 +22,18 @@ dimensions = () type = integer intent = out +[instance] + standard_name = instance_number + long_name = instance number for testing multi-instance support + units = index + dimensions = () + type = integer + intent = in [data_array] standard_name = data_array long_name = data array in km units = km - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) type = real kind = kind_phys intent = inout @@ -34,7 +41,7 @@ standard_name = data_array2 long_name = data array in m+2 s-2 units = m+2 s-2 - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) type = real kind = kind_phys intent = inout @@ -42,7 +49,7 @@ standard_name = data_array_opt long_name = optional data array in km units = km - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) type = real kind = kind_phys intent = inout diff --git a/end-to-end-tests/instances_advection/CMakeLists.txt b/end-to-end-tests/instances_advection/CMakeLists.txt new file mode 100644 index 00000000..698050e3 --- /dev/null +++ b/end-to-end-tests/instances_advection/CMakeLists.txt @@ -0,0 +1,66 @@ +#------------------------------------------------------------------------------ +# +# Multi-instance + constituents combined test +# +# Exercises per-instance state separation (from end-to-end-tests/instances) and +# host + scheme-registered constituents (boiled-down from end-to-end-tests/ +# advection). Verification is mass conservation per instance and cross-instance +# distinctness of final state. +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "cld_liq" "apply_constituent_tendencies") +set(HOST_FILES "data" "main") +set(SUITE_FILES "cld_suite.xml") +set(HOST "test_host") +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") + +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +add_executable(test_instances_advection.x + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES_FILTERED} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES}) +target_link_libraries(test_instances_advection.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_instances_advection.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_instances_advection.x PROPERTIES LINKER_LANGUAGE Fortran) + +add_test(NAME test_instances_advection + COMMAND test_instances_advection.x) diff --git a/end-to-end-tests/instances_advection/apply_constituent_tendencies.F90 b/end-to-end-tests/instances_advection/apply_constituent_tendencies.F90 new file mode 100644 index 00000000..63a1881c --- /dev/null +++ b/end-to-end-tests/instances_advection/apply_constituent_tendencies.F90 @@ -0,0 +1,39 @@ +module apply_constituent_tendencies + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: apply_constituent_tendencies_run + +contains + + !> \section arg_table_apply_constituent_tendencies_run Argument Table + !!! \htmlinclude apply_constituent_tendencies_run.html + subroutine apply_constituent_tendencies_run(const_tend, const, errcode, errmsg) + ! Dummy arguments + real(kind=kind_phys), intent(inout) :: const_tend(:, :, :) ! constituent tendency array + real(kind=kind_phys), intent(inout) :: const(:, :, :) ! constituent state array + integer, intent(out) :: errcode + character(len=512), intent(out) :: errmsg + + ! Local variables + integer :: klev, jcnst, icol + + errcode = 0 + errmsg = '' + + do icol = 1, size(const_tend, 1) + do klev = 1, size(const_tend, 2) + do jcnst = 1, size(const_tend, 3) + const(icol, klev, jcnst) = const(icol, klev, jcnst) + const_tend(icol, klev, jcnst) + end do + end do + end do + + const_tend = 0._kind_phys + + end subroutine apply_constituent_tendencies_run + +end module apply_constituent_tendencies diff --git a/end-to-end-tests/instances_advection/apply_constituent_tendencies.meta b/end-to-end-tests/instances_advection/apply_constituent_tendencies.meta new file mode 100644 index 00000000..ac02e5e4 --- /dev/null +++ b/end-to-end-tests/instances_advection/apply_constituent_tendencies.meta @@ -0,0 +1,36 @@ +##################################################################### +[ccpp-table-properties] + name = apply_constituent_tendencies + type = scheme +[ccpp-arg-table] + name = apply_constituent_tendencies_run + type = scheme +[ const_tend ] + standard_name = ccpp_constituent_tendencies + long_name = ccpp constituent tendencies + units = none + type = real | kind = kind_phys + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) + intent = inout +[ const ] + standard_name = ccpp_constituents + long_name = ccpp constituents + units = none + type = real | kind = kind_phys + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) + intent = inout +[ errcode ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + type = integer + dimensions = () + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + type = character | kind = len=512 + dimensions = () + intent = out +######################################################### diff --git a/end-to-end-tests/instances_advection/cld_liq.F90 b/end-to-end-tests/instances_advection/cld_liq.F90 new file mode 100644 index 00000000..3200505f --- /dev/null +++ b/end-to-end-tests/instances_advection/cld_liq.F90 @@ -0,0 +1,99 @@ +! Test parameterization with advected species +! + +module cld_liq + + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + implicit none + private + + public :: cld_liq_register + public :: cld_liq_init + public :: cld_liq_run + +contains + + !> \section arg_table_cld_liq_register Argument Table + !! \htmlinclude arg_table_cld_liq_register.html + !! + subroutine cld_liq_register(dyn_const, errmsg, errcode) + type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errcode + + errmsg = '' + errcode = 0 + allocate(dyn_const(1), stat=errcode) + if (errcode /= 0) then + errmsg = 'Error allocating dyn_const in cld_liq_register' + return + end if + call dyn_const(1)%instantiate(std_name="cloud_liquid_dry_mixing_ratio", long_name='Cloud liquid dry mixing ratio', & + diag_name='CLDLIQ', units='kg kg-1', default_value=0._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + mixing_ratio_type='dry', & + errcode=errcode, errmsg=errmsg) + + end subroutine cld_liq_register + + !> \section arg_table_cld_liq_run Argument Table + !! \htmlinclude arg_table_cld_liq_run.html + !! + subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, & + cld_liq_array, cld_liq_tend, errmsg, errcode) + + integer, intent(in) :: ncol + real(kind=kind_phys), intent(in) :: timestep + real(kind=kind_phys), intent(in) :: tcld + real(kind=kind_phys), intent(inout) :: temp(:, :) + real(kind=kind_phys), intent(inout) :: qv(:, :) + real(kind=kind_phys), intent(in) :: ps(:) + real(kind=kind_phys), intent(inout) :: cld_liq_array(:, :) + real(kind=kind_phys), intent(out) :: cld_liq_tend(:, :) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errcode + !---------------------------------------------------------------- + + integer :: icol + integer :: ilev + real(kind=kind_phys) :: cond + + errmsg = '' + errcode = 0 + + do icol = 1, ncol + do ilev = 1, size(temp, 2) + cld_liq_array(icol, ilev) = max(0.0_kind_phys, cld_liq_array(icol, ilev)) + if ((qv(icol, ilev) > 0.0_kind_phys) .and. & + (temp(icol, ilev) <= tcld)) then + cond = min(qv(icol, ilev), 0.1_kind_phys) + cld_liq_tend(icol, ilev) = cond + qv(icol, ilev) = qv(icol, ilev) - cond + if (cond > 0.0_kind_phys) then + temp(icol, ilev) = temp(icol, ilev) + (cond * 5.0_kind_phys) + end if + end if + end do + end do + + end subroutine cld_liq_run + + !> \section arg_table_cld_liq_init Argument Table + !! \htmlinclude arg_table_cld_liq_init.html + !! + subroutine cld_liq_init(tfreeze, tcld, errmsg, errcode) + + real(kind=kind_phys), intent(in) :: tfreeze + real(kind=kind_phys), intent(out) :: tcld + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errcode + + errmsg = '' + errcode = 0 + tcld = tfreeze - 20.0_kind_phys + + end subroutine cld_liq_init + +end module cld_liq diff --git a/end-to-end-tests/instances_advection/cld_liq.meta b/end-to-end-tests/instances_advection/cld_liq.meta new file mode 100644 index 00000000..1abc40d0 --- /dev/null +++ b/end-to-end-tests/instances_advection/cld_liq.meta @@ -0,0 +1,135 @@ +# cld_liq is a scheme that produces a cloud liquid amount +[ccpp-table-properties] + name = cld_liq + type = scheme +[ccpp-arg-table] + name = cld_liq_register + type = scheme +[ dyn_const ] + standard_name = dynamic_constituents_for_cld_liq + dimensions = (:) + type = ccpp_constituent_properties_t + intent = out + allocatable = true +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errcode ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = cld_liq_run + type = scheme +[ ncol ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ timestep ] + standard_name = time_step_for_physics + long_name = time step + units = s + dimensions = () + type = real + kind = kind_phys + intent = in +[ tcld] + standard_name = minimum_temperature_for_cloud_liquid + units = K + dimensions = () + type = real | kind = kind_phys + intent = in +[ temp ] + standard_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_LAYER_dimension) + type = real + kind = kind_phys + intent = inout +[ qv ] + standard_name = water_vapor_specific_humidity + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout +[ ps ] + standard_name = surface_air_pressure + type = real + kind = kind_phys + units = hPa + dimensions = (horizontal_dimension) + intent = in +[ cld_liq_array ] + standard_name = cloud_liquid_dry_mixing_ratio + diagnostic_name = CLDLIQ + advected = .true. + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = inout +[ cld_liq_tend ] + standard_name = tendency_of_cloud_liquid_dry_mixing_ratio + units = kg kg-1 s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = out + constituent = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errcode ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = cld_liq_init + type = scheme +[ tfreeze] + standard_name = water_temperature_at_freezing + long_name = Freezing temperature of water at sea level + units = K + dimensions = () + type = real | kind = kind_phys + intent = in +[ tcld] + standard_name = minimum_temperature_for_cloud_liquid + units = K + dimensions = () + type = real | kind = kind_phys + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errcode ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/instances_advection/cld_suite.xml b/end-to-end-tests/instances_advection/cld_suite.xml new file mode 100644 index 00000000..65e98802 --- /dev/null +++ b/end-to-end-tests/instances_advection/cld_suite.xml @@ -0,0 +1,8 @@ + + + + + cld_liq + apply_constituent_tendencies + + diff --git a/end-to-end-tests/instances_advection/data.F90 b/end-to-end-tests/instances_advection/data.F90 new file mode 100644 index 00000000..1145dc83 --- /dev/null +++ b/end-to-end-tests/instances_advection/data.F90 @@ -0,0 +1,128 @@ +module data + + use ccpp_kinds, only: kind_phys + + implicit none + public + + ! Sizing parameters (shared across instances) + integer, parameter :: ncols = 4 + integer, parameter :: pver = 3 + integer, parameter :: ninstances = 3 + + ! Time-step + freezing-point constants (shared across instances) + real(kind=kind_phys), parameter :: dt = 1.0_kind_phys + real(kind=kind_phys), parameter :: tfreeze = 273.15_kind_phys + integer, parameter :: num_time_steps = 2 + + ! qv index in the constituent state array; filled at runtime after + ! ccpp_register_constituents. Identical across instances because all + ! instances register the same constituents in the same order. + integer, protected :: index_qv = -1 + + ! Per-instance mutable physics state + ! + ! \section arg_table_physics_state Argument Table + !! \htmlinclude arg_table_physics_state.html + type physics_state + real(kind=kind_phys), allocatable :: ps(:) ! surface pressure + real(kind=kind_phys), allocatable :: temp(:, :) ! temperature + real(kind=kind_phys), pointer :: q(:, :, :) => null() ! constituent array + end type physics_state + + type(physics_state), target :: phys_state(ninstances) + + ! Per-instance distinct initial qv values (used to drive distinct results) + real(kind=kind_phys), parameter, dimension(ninstances) :: & + qv_init = (/ 1.0_kind_phys, 2.0_kind_phys, 3.0_kind_phys /) + + ! Tolerance for the final verification check + real(kind=kind_phys), parameter :: tolerance = 1.0e-12_kind_phys + +contains + + subroutine set_index_qv(idx) + integer, intent(in) :: idx + index_qv = idx + end subroutine set_index_qv + + subroutine allocate_physics_state(ins, constituents_ptr) + ! Wire phys_state(ins) to its per-instance constituent array and allocate + ! the per-instance temp/ps storage. + integer, intent(in) :: ins + real(kind=kind_phys), pointer :: constituents_ptr(:, :, :) + + if (allocated(phys_state(ins)%ps)) then + deallocate(phys_state(ins)%ps) + end if + allocate(phys_state(ins)%ps(ncols)) + phys_state(ins)%ps = 1000.0_kind_phys + + if (allocated(phys_state(ins)%temp)) then + deallocate(phys_state(ins)%temp) + end if + allocate(phys_state(ins)%temp(ncols, pver)) + ! Start cold so cld_liq_run will produce a tendency on the first call. + phys_state(ins)%temp = tfreeze - 30.0_kind_phys + + if (associated(phys_state(ins)%q)) nullify(phys_state(ins)%q) + phys_state(ins)%q => constituents_ptr + + end subroutine allocate_physics_state + + subroutine init_qv(ins) + ! Seed the per-instance constituent array with a distinct qv value. + integer, intent(in) :: ins + phys_state(ins)%q(:, :, :) = 0.0_kind_phys + phys_state(ins)%q(:, :, index_qv) = qv_init(ins) + end subroutine init_qv + + logical function verify_results(num_consts) + ! Check per-instance mass conservation and cross-instance distinctness. + integer, intent(in) :: num_consts + + real(kind=kind_phys) :: q_sum(ninstances) + real(kind=kind_phys) :: cld_liq_max(ninstances) + integer :: ins, ins2, k + logical :: ok + + verify_results = .true. + + ! Mass conservation per instance: total constituent mass per instance + ! should equal ncols * pver * qv_init(ins) (because all qv either + ! stays as qv or is moved into cld_liq by cld_liq_run; nothing else + ! produces or consumes mass in this minimal scheme set). + do ins = 1, ninstances + q_sum(ins) = 0.0_kind_phys + do k = 1, num_consts + q_sum(ins) = q_sum(ins) + sum(phys_state(ins)%q(:, :, k)) + end do + cld_liq_max(ins) = maxval(phys_state(ins)%q(:, :, :)) + ok = abs(q_sum(ins) - real(ncols, kind_phys) * real(pver, kind_phys) & + * qv_init(ins)) < tolerance * real(ncols * pver, kind_phys) & + * qv_init(ins) + if (.not. ok) then + write(6, '(a,i0,a,es15.7,a,es15.7)') & + 'FAIL mass conservation for instance ', ins, & + ': q_sum=', q_sum(ins), ' expected~', & + real(ncols, kind_phys) * real(pver, kind_phys) * qv_init(ins) + verify_results = .false. + end if + end do + + ! Cross-instance distinctness: instance i must end up with a different + ! state than instance j, since they started from different qv. + do ins = 1, ninstances + do ins2 = ins + 1, ninstances + if (abs(q_sum(ins) - q_sum(ins2)) < tolerance) then + write(6, '(a,i0,a,i0,a)') & + 'FAIL distinctness: instance ', ins, ' and instance ', ins2, & + ' have identical totals (state leaked between instances?)' + verify_results = .false. + end if + end do + end do + + end function verify_results + +end module data diff --git a/end-to-end-tests/instances_advection/data.meta b/end-to-end-tests/instances_advection/data.meta new file mode 100644 index 00000000..f860b346 --- /dev/null +++ b/end-to-end-tests/instances_advection/data.meta @@ -0,0 +1,83 @@ +[ccpp-table-properties] + name = physics_state + type = ddt +[ccpp-arg-table] + name = physics_state + type = ddt +[ps] + standard_name = surface_air_pressure + units = hPa + dimensions = (horizontal_dimension) + type = real + kind = kind_phys +[temp] + standard_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys +[q] + standard_name = state_constituent_mixing_ratio + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) + type = real + kind = kind_phys +[q(:,:,index_of_water_vapor_specific_humidity)] + standard_name = water_vapor_specific_humidity + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + + +[ccpp-table-properties] + name = data + type = host + dependencies = + +[ccpp-arg-table] + name = data + type = host +[ncols] + standard_name = horizontal_dimension + long_name = horizontal dimension + units = count + dimensions = () + type = integer + protected = True +[pver] + standard_name = vertical_layer_dimension + long_name = vertical layer dimension + units = count + dimensions = () + type = integer + protected = True +[dt] + standard_name = time_step_for_physics + long_name = time step for physics + units = s + dimensions = () + type = real + kind = kind_phys + protected = True +[tfreeze] + standard_name = water_temperature_at_freezing + long_name = freezing temperature of water at sea level + units = K + dimensions = () + type = real + kind = kind_phys + protected = True +[index_qv] + standard_name = index_of_water_vapor_specific_humidity + long_name = index of water vapor specific humidity in the constituent array + units = index + dimensions = () + type = integer + protected = True +[phys_state] + standard_name = physics_state_derived_type + long_name = per-instance physics state DDT + units = none + dimensions = (number_of_instances) + type = physics_state diff --git a/end-to-end-tests/instances_advection/main.F90 b/end-to-end-tests/instances_advection/main.F90 new file mode 100644 index 00000000..2a1e1c84 --- /dev/null +++ b/end-to-end-tests/instances_advection/main.F90 @@ -0,0 +1,256 @@ +program test_instances_advection + + use, intrinsic :: iso_fortran_env, only: error_unit +#ifdef _OPENMP + use omp_lib +#endif + + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + use data, only: ncols, pver, ninstances, dt, tfreeze, num_time_steps + use data, only: phys_state, qv_init, index_qv + use data, only: allocate_physics_state, init_qv, set_index_qv, & + verify_results + + use test_host_ccpp_cap, only: ccpp_register, ccpp_init, ccpp_final + use test_host_ccpp_cap, only: ccpp_physics_init, ccpp_physics_run, & + ccpp_physics_timestep_init, ccpp_physics_timestep_final, & + ccpp_physics_final + use test_host_ccpp_cap, only: ccpp_register_constituents, & + ccpp_initialize_constituents, ccpp_deallocate_dynamic_constituents + use test_host_ccpp_cap, only: ccpp_const_get_index, ccpp_constituents_array + use test_host_ccpp_cap, only: ccpp_number_constituents + + implicit none + + character(len=*), parameter :: ccpp_suite = 'cld_suite' + character(len=512) :: errmsg + integer :: errcode + integer :: nphys_threads + integer :: ins + integer :: tstep + integer :: num_consts + integer :: idx + type(ccpp_constituent_properties_t), target, allocatable :: host_consts(:) + real(kind=kind_phys), pointer :: constituents_ptr(:, :, :) + + ! Use OpenMP threading in physics (internally) where available. +#ifdef _OPENMP + nphys_threads = omp_get_max_threads() +#else + nphys_threads = 1 +#endif + + !----------------------------------------------------------------- + ! 1. CCPP register: per-instance, populates per-suite dynamic + ! constituent buffer on first instance, idempotent thereafter. + !----------------------------------------------------------------- + do ins = 1, ninstances + call ccpp_register(suite_name=ccpp_suite, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(error_unit, '(2a)') 'ccpp_register failed: ', trim(errmsg) + stop 1 + end if + end do + + !----------------------------------------------------------------- + ! 2. Build the host-registered constituent list (specific_humidity) + ! and call ccpp_register_constituents per instance. Each call + ! consumes the host_consts array (new_field sets const_index on + ! the input objects), so we re-instantiate per instance. + !----------------------------------------------------------------- + do ins = 1, ninstances + if (allocated(host_consts)) deallocate(host_consts) + allocate(host_consts(1)) + call host_consts(1)%instantiate( & + std_name='water_vapor_specific_humidity', & + long_name='Water vapor specific humidity', & + diag_name='QV', units='kg kg-1', & + vertical_dim='vertical_layer_dimension', advected=.true., & + default_value=0.0_kind_phys, mixing_ratio_type='wet', & + errcode=errcode, errmsg=errmsg) + if (errcode /= 0) then + write(error_unit, '(2a)') 'instantiate failed: ', trim(errmsg) + stop 1 + end if + call ccpp_register_constituents(host_constituents=host_consts, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(error_unit, '(a,i0,2a)') & + 'ccpp_register_constituents failed for instance ', ins, & + ': ', trim(errmsg) + stop 1 + end if + end do + + !----------------------------------------------------------------- + ! 3. ccpp_initialize_constituents per instance — allocates the + ! per-instance constituent storage. + !----------------------------------------------------------------- + do ins = 1, ninstances + call ccpp_initialize_constituents(ncols=ncols, num_layers=pver, & + instance=ins, errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(error_unit, '(a,i0,2a)') & + 'ccpp_initialize_constituents failed for instance ', ins, & + ': ', trim(errmsg) + stop 1 + end if + end do + + !----------------------------------------------------------------- + ! 4. Resolve the qv constituent index and number of constituents + ! (identical across instances). + !----------------------------------------------------------------- + + call ccpp_const_get_index(stdname='water_vapor_specific_humidity', & + const_index=idx, instance=1, errcode=errcode, errmsg=errmsg) + if (errcode /= 0) then + write(error_unit, '(2a)') 'ccpp_const_get_index(qv) failed: ', & + trim(errmsg) + stop 1 + end if + call set_index_qv(idx) + + call ccpp_number_constituents(num_flds=num_consts, instance=1, & + errcode=errcode, errmsg=errmsg) + if (errcode /= 0) then + write(error_unit, '(2a)') 'ccpp_number_constituents failed: ', & + trim(errmsg) + stop 1 + end if + + !----------------------------------------------------------------- + ! 5. For each instance, wire phys_state(ins) to its constituent + ! storage and seed distinct initial qv values. + !----------------------------------------------------------------- + + do ins = 1, ninstances + constituents_ptr => ccpp_constituents_array(ins) + call allocate_physics_state(ins, constituents_ptr) + call init_qv(ins) + end do + + !----------------------------------------------------------------- + ! 6. ccpp_init and ccpp_physics_init per instance. + !----------------------------------------------------------------- + do ins = 1, ninstances + call ccpp_init(suite_name=ccpp_suite, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(error_unit, '(a,i0,2a)') 'ccpp_init failed for instance ', & + ins, ': ', trim(errmsg) + stop 1 + end if + end do + do ins = 1, ninstances + call ccpp_physics_init(suite_name=ccpp_suite, group_name='physics', & + lb=1, ub=ncols, thread_num=1, nthreads=1, & + nphys_threads=nphys_threads, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(error_unit, '(a,i0,2a)') & + 'ccpp_physics_init failed for instance ', ins, ': ', trim(errmsg) + stop 1 + end if + end do + + !----------------------------------------------------------------- + ! 7. Timestep loop. + !----------------------------------------------------------------- + do tstep = 1, num_time_steps + do ins = 1, ninstances + call ccpp_physics_timestep_init(suite_name=ccpp_suite, & + group_name='physics', lb=1, ub=ncols, & + thread_num=1, nthreads=1, nphys_threads=nphys_threads, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(error_unit, '(a,i0,2a)') & + 'ccpp_physics_timestep_init failed for instance ', ins, & + ': ', trim(errmsg) + stop 1 + end if + end do + do ins = 1, ninstances + call ccpp_physics_run(suite_name=ccpp_suite, group_name='physics', & + lb=1, ub=ncols, thread_num=1, nthreads=1, & + nphys_threads=nphys_threads, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(error_unit, '(a,i0,2a)') & + 'ccpp_physics_run failed for instance ', ins, ': ', trim(errmsg) + stop 1 + end if + end do + do ins = 1, ninstances + call ccpp_physics_timestep_final(suite_name=ccpp_suite, & + group_name='physics', lb=1, ub=ncols, & + thread_num=1, nthreads=1, nphys_threads=nphys_threads, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(error_unit, '(a,i0,2a)') & + 'ccpp_physics_timestep_final failed for instance ', ins, & + ': ', trim(errmsg) + stop 1 + end if + end do + end do + + !----------------------------------------------------------------- + ! 8. ccpp_physics_final per instance. + !----------------------------------------------------------------- + do ins = 1, ninstances + call ccpp_physics_final(suite_name=ccpp_suite, group_name='physics', & + lb=1, ub=ncols, thread_num=1, nthreads=1, & + nphys_threads=nphys_threads, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(error_unit, '(a,i0,2a)') & + 'ccpp_physics_final failed for instance ', ins, ': ', trim(errmsg) + stop 1 + end if + end do + + !----------------------------------------------------------------- + ! 9. Verify per-instance results BEFORE constituent teardown. + ! phys_state(:)%q points into the framework's per-instance + ! constituent storage; ccpp_deallocate_dynamic_constituents + ! frees that storage, so any verification must run first. + !----------------------------------------------------------------- + if (.not. verify_results(num_consts)) then + write(6, '(a)') 'FAIL: per-instance + constituents test' + stop 1 + end if + + !----------------------------------------------------------------- + ! 10. Teardown. + !----------------------------------------------------------------- + do ins = 1, ninstances + call ccpp_final(suite_name=ccpp_suite, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then + write(error_unit, '(a,i0,2a)') 'ccpp_final failed for instance ', & + ins, ': ', trim(errmsg) + stop 1 + end if + end do + do ins = 1, ninstances + call ccpp_deallocate_dynamic_constituents(instance=ins) + end do + deallocate(host_consts) + + write(6, '(a)') 'PASS: per-instance + constituents test' + stop 0 + +end program test_instances_advection diff --git a/end-to-end-tests/instances_advection/main.meta b/end-to-end-tests/instances_advection/main.meta new file mode 100644 index 00000000..5857002a --- /dev/null +++ b/end-to-end-tests/instances_advection/main.meta @@ -0,0 +1,77 @@ +[ccpp-table-properties] + name = main + type = control + dependencies = + +[ccpp-arg-table] + name = main + type = control +[suite_name] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[group_name] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[lb] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ub] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[thread_num] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[nthreads] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[nphys_threads] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer +[errmsg] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[errcode] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer +[instance] + standard_name = instance_number + long_name = current model instance number + units = index + dimensions = () + type = integer +[ninstances] + standard_name = number_of_instances + long_name = number of instances for multi-instance test + units = count + dimensions = () + type = integer diff --git a/end-to-end-tests/nested_suite/CMakeLists.txt b/end-to-end-tests/nested_suite/CMakeLists.txt new file mode 100644 index 00000000..5903fcb9 --- /dev/null +++ b/end-to-end-tests/nested_suite/CMakeLists.txt @@ -0,0 +1,75 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "effr_calc" "effrs_calc" "effr_diag" "effr_pre" "effr_post" "rad_lw" "rad_sw" "suite_lifecycle") +set(HOST_FILES "module_rad_ddt" "test_host_data" "test_host_mod" "test_host") +set(SUITE_FILES "main_suite.xml") +set(HOST "test_host") +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") + +# Run ccpp_capgen +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +set(EXTRA_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/../utils/test_utils.F90 +) + +add_executable(test_nested_suite.x + ${EXTRA_FILES} + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES_FILTERED} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES} + test_nested_suite_integration.F90 +) +target_link_libraries(test_nested_suite.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_nested_suite.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_nested_suite.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_nested_suite + COMMAND test_nested_suite.x) diff --git a/test/nested_suite_test/README.md b/end-to-end-tests/nested_suite/README.md similarity index 91% rename from test/nested_suite_test/README.md rename to end-to-end-tests/nested_suite/README.md index d3c3182f..5723560b 100644 --- a/test/nested_suite_test/README.md +++ b/end-to-end-tests/nested_suite/README.md @@ -5,6 +5,7 @@ Tests the capability to process nested suites: - Perform same tests as variable compatibility test at that date - Parse new XML schema 2.0 - Expand nested suites at the group level and inside groups +- Test single init and final schemes in suite ## Building/Running diff --git a/test/nested_suite_test/effr_calc.F90 b/end-to-end-tests/nested_suite/effr_calc.F90 similarity index 100% rename from test/nested_suite_test/effr_calc.F90 rename to end-to-end-tests/nested_suite/effr_calc.F90 diff --git a/test/nested_suite_test/effr_calc.meta b/end-to-end-tests/nested_suite/effr_calc.meta similarity index 87% rename from test/nested_suite_test/effr_calc.meta rename to end-to-end-tests/nested_suite/effr_calc.meta index c3733f13..6361eac6 100644 --- a/test/nested_suite_test/effr_calc.meta +++ b/end-to-end-tests/nested_suite/effr_calc.meta @@ -33,7 +33,7 @@ name = effr_calc_run type = scheme [ ncol ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension type = integer units = count dimensions = () @@ -48,7 +48,7 @@ standard_name = effective_radius_of_stratiform_cloud_rain_particle long_name = effective radius of cloud rain particle in micrometer units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = in @@ -57,7 +57,7 @@ standard_name = effective_radius_of_stratiform_cloud_graupel long_name = effective radius of cloud graupel in micrometer units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = in @@ -66,7 +66,7 @@ standard_name = cloud_graupel_number_concentration long_name = number concentration of cloud graupel units = kg-1 - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = in @@ -75,7 +75,7 @@ standard_name = cloud_ice_number_concentration long_name = number concentration of cloud ice units = kg-1 - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = out @@ -84,7 +84,7 @@ standard_name = effective_radius_of_stratiform_cloud_liquid_water_particle long_name = effective radius of cloud liquid water particle in micrometer units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = inout @@ -92,7 +92,7 @@ standard_name = effective_radius_of_stratiform_cloud_ice_particle long_name = effective radius of cloud ice water particle in micrometer units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = out @@ -101,7 +101,7 @@ standard_name = effective_radius_of_stratiform_cloud_snow_particle long_name = effective radius of cloud snow particle in micrometer units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = 8 intent = inout @@ -110,7 +110,7 @@ standard_name = cloud_liquid_number_concentration long_name = number concentration of cloud liquid units = kg-1 - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = out diff --git a/test/nested_suite_test/effr_diag.F90 b/end-to-end-tests/nested_suite/effr_diag.F90 similarity index 100% rename from test/nested_suite_test/effr_diag.F90 rename to end-to-end-tests/nested_suite/effr_diag.F90 diff --git a/test/var_compatibility_test/effr_diag.meta b/end-to-end-tests/nested_suite/effr_diag.meta similarity index 96% rename from test/var_compatibility_test/effr_diag.meta rename to end-to-end-tests/nested_suite/effr_diag.meta index 9e0e4fc2..5a5c9e67 100644 --- a/test/var_compatibility_test/effr_diag.meta +++ b/end-to-end-tests/nested_suite/effr_diag.meta @@ -36,7 +36,7 @@ standard_name = effective_radius_of_stratiform_cloud_rain_particle long_name = effective radius of cloud rain particle in micrometer units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = in diff --git a/test/nested_suite_test/effr_post.F90 b/end-to-end-tests/nested_suite/effr_post.F90 similarity index 100% rename from test/nested_suite_test/effr_post.F90 rename to end-to-end-tests/nested_suite/effr_post.F90 diff --git a/test/nested_suite_test/effr_post.meta b/end-to-end-tests/nested_suite/effr_post.meta similarity index 96% rename from test/nested_suite_test/effr_post.meta rename to end-to-end-tests/nested_suite/effr_post.meta index 721582a6..703b5ebc 100644 --- a/test/nested_suite_test/effr_post.meta +++ b/end-to-end-tests/nested_suite/effr_post.meta @@ -36,7 +36,7 @@ standard_name = effective_radius_of_stratiform_cloud_rain_particle long_name = effective radius of cloud rain particle in micrometer units = m - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = inout diff --git a/test/nested_suite_test/effr_pre.F90 b/end-to-end-tests/nested_suite/effr_pre.F90 similarity index 100% rename from test/nested_suite_test/effr_pre.F90 rename to end-to-end-tests/nested_suite/effr_pre.F90 diff --git a/test/nested_suite_test/effr_pre.meta b/end-to-end-tests/nested_suite/effr_pre.meta similarity index 96% rename from test/nested_suite_test/effr_pre.meta rename to end-to-end-tests/nested_suite/effr_pre.meta index 251b4175..c47d1abf 100644 --- a/test/nested_suite_test/effr_pre.meta +++ b/end-to-end-tests/nested_suite/effr_pre.meta @@ -37,7 +37,7 @@ standard_name = effective_radius_of_stratiform_cloud_rain_particle long_name = effective radius of cloud rain particle in micrometer units = m - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = inout diff --git a/test/nested_suite_test/effrs_calc.F90 b/end-to-end-tests/nested_suite/effrs_calc.F90 similarity index 100% rename from test/nested_suite_test/effrs_calc.F90 rename to end-to-end-tests/nested_suite/effrs_calc.F90 diff --git a/test/nested_suite_test/effrs_calc.meta b/end-to-end-tests/nested_suite/effrs_calc.meta similarity index 88% rename from test/nested_suite_test/effrs_calc.meta rename to end-to-end-tests/nested_suite/effrs_calc.meta index 9ce7b88e..e2fd1de9 100644 --- a/test/nested_suite_test/effrs_calc.meta +++ b/end-to-end-tests/nested_suite/effrs_calc.meta @@ -9,7 +9,7 @@ standard_name = effective_radius_of_stratiform_cloud_snow_particle units = m type = real | kind = kind_phys - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) intent = inout [ errmsg ] standard_name = ccpp_error_message diff --git a/test/nested_suite_test/main_suite.xml b/end-to-end-tests/nested_suite/main_suite.xml similarity index 90% rename from test/nested_suite_test/main_suite.xml rename to end-to-end-tests/nested_suite/main_suite.xml index a319ec47..be2d0d07 100644 --- a/test/nested_suite_test/main_suite.xml +++ b/end-to-end-tests/nested_suite/main_suite.xml @@ -1,6 +1,7 @@ + suite_lifecycle effr_pre @@ -15,4 +16,5 @@ + suite_lifecycle diff --git a/test/nested_suite_test/module_rad_ddt.F90 b/end-to-end-tests/nested_suite/module_rad_ddt.F90 similarity index 100% rename from test/nested_suite_test/module_rad_ddt.F90 rename to end-to-end-tests/nested_suite/module_rad_ddt.F90 diff --git a/test/nested_suite_test/module_rad_ddt.meta b/end-to-end-tests/nested_suite/module_rad_ddt.meta similarity index 100% rename from test/nested_suite_test/module_rad_ddt.meta rename to end-to-end-tests/nested_suite/module_rad_ddt.meta diff --git a/test/nested_suite_test/rad_lw.F90 b/end-to-end-tests/nested_suite/rad_lw.F90 similarity index 100% rename from test/nested_suite_test/rad_lw.F90 rename to end-to-end-tests/nested_suite/rad_lw.F90 diff --git a/test/nested_suite_test/rad_lw.meta b/end-to-end-tests/nested_suite/rad_lw.meta similarity index 89% rename from test/nested_suite_test/rad_lw.meta rename to end-to-end-tests/nested_suite/rad_lw.meta index 883edf1b..bfab7426 100644 --- a/test/nested_suite_test/rad_lw.meta +++ b/end-to-end-tests/nested_suite/rad_lw.meta @@ -6,7 +6,7 @@ name = rad_lw_run type = scheme [ ncol ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension type = integer units = count dimensions = () @@ -15,7 +15,7 @@ standard_name = longwave_radiation_fluxes long_name = longwave radiation fluxes units = W m-2 - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) type = ty_rad_lw intent = inout [ errmsg ] diff --git a/test/nested_suite_test/rad_sw.F90 b/end-to-end-tests/nested_suite/rad_sw.F90 similarity index 100% rename from test/nested_suite_test/rad_sw.F90 rename to end-to-end-tests/nested_suite/rad_sw.F90 diff --git a/test/var_compatibility_test/rad_sw.meta b/end-to-end-tests/nested_suite/rad_sw.meta similarity index 87% rename from test/var_compatibility_test/rad_sw.meta rename to end-to-end-tests/nested_suite/rad_sw.meta index d88b9acc..af88530f 100644 --- a/test/var_compatibility_test/rad_sw.meta +++ b/end-to-end-tests/nested_suite/rad_sw.meta @@ -5,7 +5,7 @@ name = rad_sw_run type = scheme [ ncol ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension type = integer units = count dimensions = () @@ -13,14 +13,14 @@ [ sfc_up_sw ] standard_name = surface_upwelling_shortwave_radiation_flux units = W m2 - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) type = real kind = kind_phys intent = inout [ sfc_down_sw ] standard_name = surface_downwelling_shortwave_radiation_flux units = W m2 - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) type = real kind = kind_phys intent = inout diff --git a/test/nested_suite_test/radiation2_suite.xml b/end-to-end-tests/nested_suite/radiation2_suite.xml similarity index 100% rename from test/nested_suite_test/radiation2_suite.xml rename to end-to-end-tests/nested_suite/radiation2_suite.xml diff --git a/test/nested_suite_test/radiation3_subsuite.xml b/end-to-end-tests/nested_suite/radiation3_subsuite.xml similarity index 100% rename from test/nested_suite_test/radiation3_subsuite.xml rename to end-to-end-tests/nested_suite/radiation3_subsuite.xml diff --git a/test/nested_suite_test/radiation3_suite.xml b/end-to-end-tests/nested_suite/radiation3_suite.xml similarity index 100% rename from test/nested_suite_test/radiation3_suite.xml rename to end-to-end-tests/nested_suite/radiation3_suite.xml diff --git a/test/nested_suite_test/radiation4_suite.xml b/end-to-end-tests/nested_suite/radiation4_suite.xml similarity index 100% rename from test/nested_suite_test/radiation4_suite.xml rename to end-to-end-tests/nested_suite/radiation4_suite.xml diff --git a/end-to-end-tests/nested_suite/suite_lifecycle.F90 b/end-to-end-tests/nested_suite/suite_lifecycle.F90 new file mode 100644 index 00000000..09b85670 --- /dev/null +++ b/end-to-end-tests/nested_suite/suite_lifecycle.F90 @@ -0,0 +1,34 @@ +module suite_lifecycle + + implicit none + private + + public :: suite_lifecycle_init, suite_lifecycle_final + +contains + + !> \section arg_table_suite_lifecycle_init Argument Table + !! \htmlinclude arg_table_suite_lifecycle_init.html + !! + subroutine suite_lifecycle_init(counter, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(inout) :: counter + errmsg = '' + errflg = 0 + counter = counter + 1 + end subroutine suite_lifecycle_init + + !> \section arg_table_suite_lifecycle_final Argument Table + !! \htmlinclude arg_table_suite_lifecycle_final.html + !! + subroutine suite_lifecycle_final(counter, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(inout) :: counter + errmsg = '' + errflg = 0 + counter = counter + 1 + end subroutine suite_lifecycle_final + +end module suite_lifecycle diff --git a/end-to-end-tests/nested_suite/suite_lifecycle.meta b/end-to-end-tests/nested_suite/suite_lifecycle.meta new file mode 100644 index 00000000..089ffd6e --- /dev/null +++ b/end-to-end-tests/nested_suite/suite_lifecycle.meta @@ -0,0 +1,49 @@ +[ccpp-table-properties] + name = suite_lifecycle + type = scheme + +[ccpp-arg-table] + name = suite_lifecycle_init + type = scheme +[counter] + standard_name = lifecycle_counter + units = 1 + dimensions = () + type = integer + intent = inout +[errmsg] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = suite_lifecycle_final + type = scheme +[counter] + standard_name = lifecycle_counter + units = 1 + dimensions = () + type = integer + intent = inout +[errmsg] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/test/var_compatibility_test/test_host.F90 b/end-to-end-tests/nested_suite/test_host.F90 similarity index 68% rename from test/var_compatibility_test/test_host.F90 rename to end-to-end-tests/nested_suite/test_host.F90 index 67c7a1ac..0e3bade7 100644 --- a/test/var_compatibility_test/test_host.F90 +++ b/end-to-end-tests/nested_suite/test_host.F90 @@ -104,11 +104,14 @@ end function check_suite subroutine test_host(retval, test_suites) use test_host_mod, only: ncols - use test_host_ccpp_cap, only: test_host_ccpp_physics_initialize - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_initial - use test_host_ccpp_cap, only: test_host_ccpp_physics_run - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_final - use test_host_ccpp_cap, only: test_host_ccpp_physics_finalize + use test_host_ccpp_cap, only: ccpp_register + use test_host_ccpp_cap, only: ccpp_init + use test_host_ccpp_cap, only: ccpp_physics_init + use test_host_ccpp_cap, only: ccpp_physics_timestep_init + use test_host_ccpp_cap, only: ccpp_physics_run + use test_host_ccpp_cap, only: ccpp_physics_timestep_final + use test_host_ccpp_cap, only: ccpp_physics_final + use test_host_ccpp_cap, only: ccpp_final use test_host_ccpp_cap, only: ccpp_physics_suite_list use test_host_mod, only: init_data, & compare_data @@ -155,10 +158,35 @@ subroutine test_host(retval, test_suites) return end if + ! Use the suite information to register + do sind = 1, num_suites + call ccpp_register( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in initialize of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + end if + end do + + ! Use the suite information to initialize + do sind = 1, num_suites + call ccpp_init( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in initialize of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + end if + end do + ! Use the suite information to setup the run do sind = 1, num_suites - call test_host_ccpp_physics_initialize(test_suites(sind)%suite_name, & - errmsg, errflg) + call ccpp_physics_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) if (errflg /= 0) then write(6, '(4a)') 'ERROR in initialize of ', & trim(test_suites(sind)%suite_name), ': ', trim(errmsg) @@ -171,15 +199,15 @@ subroutine test_host(retval, test_suites) exit end if if (errflg == 0) then - call test_host_ccpp_physics_timestep_initial( & - test_suites(sind)%suite_name, errmsg, errflg) - end if - if (errflg /= 0) then - write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & - trim(errmsg) - exit + call ccpp_physics_timestep_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) end if if (errflg /= 0) then + write(6, '(2a)') 'An error occurred in ccpp_physics_timestep_init, ', & + 'Exiting...' exit end if end do @@ -199,10 +227,12 @@ subroutine test_host(retval, test_suites) exit end if if (errflg == 0) then - call test_host_ccpp_physics_run( & - test_suites(sind)%suite_name, & - test_suites(sind)%suite_parts(index), & - col_start, col_end, errmsg, errflg) + call ccpp_physics_run( & + suite_name=test_suites(sind)%suite_name, & + group_name=test_suites(sind)%suite_parts(index), & + col_start=col_start, col_end=col_end, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) end if if (errflg /= 0) then write(6, '(5a)') trim(test_suites(sind)%suite_name), & @@ -219,12 +249,33 @@ subroutine test_host(retval, test_suites) exit end if if (errflg == 0) then - call test_host_ccpp_physics_timestep_final( & - test_suites(sind)%suite_name, errmsg, errflg) + call ccpp_physics_timestep_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) end if if (errflg /= 0) then - write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & - trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_physics_timestep_final, ', & + 'Exiting...' + exit + end if + end do + + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_physics_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + end if + if (errflg /= 0) then + write(6, '(2a)') 'An error occurred in ccpp_physics_final, ', & + 'Exiting...' exit end if end do @@ -234,13 +285,12 @@ subroutine test_host(retval, test_suites) exit end if if (errflg == 0) then - call test_host_ccpp_physics_finalize( & - test_suites(sind)%suite_name, errmsg, errflg) + call ccpp_final( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) end if if (errflg /= 0) then - write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & - trim(errmsg) - write(6, '(2a)') 'An error occurred in ccpp_timestep_final, ', & + write(6, '(2a)') 'An error occurred in ccpp_final, ', & 'Exiting...' exit end if diff --git a/end-to-end-tests/nested_suite/test_host.meta b/end-to-end-tests/nested_suite/test_host.meta new file mode 100644 index 00000000..076cd7d6 --- /dev/null +++ b/end-to-end-tests/nested_suite/test_host.meta @@ -0,0 +1,63 @@ +[ccpp-table-properties] + name = test_host + type = control +[ccpp-arg-table] + name = test_host + type = control +[ col_start ] + standard_name = horizontal_loop_begin + type = integer + units = count + dimensions = () + protected = True +[ col_end ] + standard_name = horizontal_loop_end + type = integer + units = count + dimensions = () + protected = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer diff --git a/test/nested_suite_test/test_host_data.F90 b/end-to-end-tests/nested_suite/test_host_data.F90 similarity index 100% rename from test/nested_suite_test/test_host_data.F90 rename to end-to-end-tests/nested_suite/test_host_data.F90 diff --git a/test/nested_suite_test/test_host_data.meta b/end-to-end-tests/nested_suite/test_host_data.meta similarity index 94% rename from test/nested_suite_test/test_host_data.meta rename to end-to-end-tests/nested_suite/test_host_data.meta index 59a0fb4d..2154b797 100644 --- a/test/nested_suite_test/test_host_data.meta +++ b/end-to-end-tests/nested_suite/test_host_data.meta @@ -42,7 +42,6 @@ dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys - intent = in active = (flag_indicating_cloud_microphysics_has_graupel) [nci] standard_name = cloud_ice_number_concentration @@ -51,7 +50,6 @@ dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys - intent = in active = (flag_indicating_cloud_microphysics_has_ice) [scalar_var] standard_name = scalar_variable_for_testing @@ -119,10 +117,11 @@ dimensions = () type = integer -[ccpp-table-properties] - name = test_host_data - type = module - dependencies = module_rad_ddt.F90 -[ccpp-arg-table] - name = test_host_data - type = module +#[ccpp-table-properties] +# name = test_host_data +# type = host +# dependencies = module_rad_ddt.F90 +#[ccpp-arg-table] +# name = test_host_data +# type = host +# \ No newline at end of file diff --git a/test/nested_suite_test/test_host_mod.F90 b/end-to-end-tests/nested_suite/test_host_mod.F90 similarity index 93% rename from test/nested_suite_test/test_host_mod.F90 rename to end-to-end-tests/nested_suite/test_host_mod.F90 index d3bde866..dfe95d31 100644 --- a/test/nested_suite_test/test_host_mod.F90 +++ b/end-to-end-tests/nested_suite/test_host_mod.F90 @@ -12,10 +12,11 @@ module test_host_mod !! integer, parameter :: ncols = 12 integer, parameter :: pver = 4 - type(physics_state) :: phys_state + type(physics_state), target :: phys_state real(kind=kind_phys) :: effrs(ncols, pver) logical, parameter :: has_ice = .true. logical, parameter :: has_graupel = .true. + integer :: lifecycle_counter public :: init_data public :: compare_data @@ -43,6 +44,7 @@ subroutine init_data() end if phys_state%tke = 10.0 !J kg-1 phys_state%tke2 = 42.0 !J kg-1 + lifecycle_counter = 0 end subroutine init_data @@ -59,6 +61,7 @@ logical function compare_data() real(kind=kind_phys), parameter :: sfc_down_sw_expected = 400. ! W/m2 real(kind=kind_phys), parameter :: sfc_up_lw_expected = 300. ! W/m2 real(kind=kind_phys), parameter :: sfc_down_lw_expected = 50. ! W/m2 + integer, parameter :: lifecycle_counter_expected = 2 compare_data = .true. @@ -127,6 +130,12 @@ logical function compare_data() compare_data = .false. end if + if (lifecycle_counter /= lifecycle_counter_expected) then + write(6, '(a,i0,a,i0)') & + 'Error: lifecycle_counter does not match expected value: ', lifecycle_counter, ' vs ', lifecycle_counter_expected + compare_data = .false. + end if + end function compare_data end module test_host_mod diff --git a/test/var_compatibility_test/test_host_mod.meta b/end-to-end-tests/nested_suite/test_host_mod.meta similarity index 89% rename from test/var_compatibility_test/test_host_mod.meta rename to end-to-end-tests/nested_suite/test_host_mod.meta index 51a2f5c3..ab90ebb2 100644 --- a/test/var_compatibility_test/test_host_mod.meta +++ b/end-to-end-tests/nested_suite/test_host_mod.meta @@ -1,9 +1,9 @@ [ccpp-table-properties] name = test_host_mod - type = module + type = host [ccpp-arg-table] name = test_host_mod - type = module + type = host [ ncols] standard_name = horizontal_dimension units = count @@ -40,3 +40,8 @@ units = flag dimensions = () type = logical +[lifecycle_counter] + standard_name = lifecycle_counter + units = 1 + dimensions = () + type = integer diff --git a/test/nested_suite_test/test_nested_suite_integration.F90 b/end-to-end-tests/nested_suite/test_nested_suite_integration.F90 similarity index 100% rename from test/nested_suite_test/test_nested_suite_integration.F90 rename to end-to-end-tests/nested_suite/test_nested_suite_integration.F90 diff --git a/end-to-end-tests/opt_arg/CMakeLists.txt b/end-to-end-tests/opt_arg/CMakeLists.txt new file mode 100644 index 00000000..d791fa28 --- /dev/null +++ b/end-to-end-tests/opt_arg/CMakeLists.txt @@ -0,0 +1,66 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "opt_arg_scheme") +set(HOST_FILES "data" "main") +set(SUITE_FILES "suite_opt_arg_suite.xml") +set(HOST "test_host") +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") + +# Run ccpp_capgen. Override kind_phys to REAL32 so the whole test +# runs in single precision; exercises the --kind-type plumbing. +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + KIND_SPECS "kind_phys=REAL32" + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +add_executable(test_opt_arg.x ${SCHEME_FORTRAN_FILES_FILTERED} ${HOST_FORTRAN_FILES} ${CAPGEN_FILES}) +target_link_libraries(test_opt_arg.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_opt_arg.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_opt_arg.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_opt_arg + COMMAND test_opt_arg.x) diff --git a/test_prebuild/test_opt_arg/data.F90 b/end-to-end-tests/opt_arg/data.F90 similarity index 52% rename from test_prebuild/test_opt_arg/data.F90 rename to end-to-end-tests/opt_arg/data.F90 index f66cf8c1..621bb428 100644 --- a/test_prebuild/test_opt_arg/data.F90 +++ b/end-to-end-tests/opt_arg/data.F90 @@ -3,21 +3,19 @@ module data !! \section arg_table_data Argument Table !! \htmlinclude data.html !! - use ccpp_types, only: ccpp_t use ccpp_kinds, only: kind_phys implicit none private - public cdata, nx, flag_for_opt_arg, std_arg, opt_arg, opt_arg_2 + public nx, flag_for_opt_arg, std_arg, opt_arg, opt_arg_2 - type(ccpp_t), target :: cdata integer, parameter :: nx = 3 logical :: flag_for_opt_arg integer, dimension(nx) :: std_arg - integer, dimension(:), allocatable :: opt_arg - real(kind=kind_phys), dimension(:), allocatable :: opt_arg_2 + integer, dimension(:), allocatable, target :: opt_arg + real(kind=kind_phys), dimension(:), allocatable, target :: opt_arg_2 end module data diff --git a/test_prebuild/test_opt_arg/data.meta b/end-to-end-tests/opt_arg/data.meta similarity index 82% rename from test_prebuild/test_opt_arg/data.meta rename to end-to-end-tests/opt_arg/data.meta index 03f3c472..abbc2086 100644 --- a/test_prebuild/test_opt_arg/data.meta +++ b/end-to-end-tests/opt_arg/data.meta @@ -1,16 +1,10 @@ [ccpp-table-properties] name = data - type = module + type = host dependencies = [ccpp-arg-table] name = data - type = module -[cdata] - standard_name = ccpp_t_instance - long_name = instance of derived data type ccpp_t - units = DDT - dimensions = () - type = ccpp_t + type = host [nx] standard_name = size_of_std_arg long_name = size of std_arg diff --git a/end-to-end-tests/opt_arg/main.F90 b/end-to-end-tests/opt_arg/main.F90 new file mode 100644 index 00000000..e0643513 --- /dev/null +++ b/end-to-end-tests/opt_arg/main.F90 @@ -0,0 +1,156 @@ +program test_opt_arg + + use, intrinsic :: iso_fortran_env, only: output_unit, & + error_unit + + use data, only: nx, & + flag_for_opt_arg, & + std_arg, & + opt_arg, & + opt_arg_2 + + use test_host_ccpp_cap, only: ccpp_register, & + ccpp_init, & + ccpp_physics_init, & + ccpp_physics_timestep_init, & + ccpp_physics_run, & + ccpp_physics_timestep_final, & + ccpp_physics_final, & + ccpp_final + + implicit none + + character(len=*), parameter :: ccpp_suite = 'opt_arg_suite' + character(len=512) :: errmsg + integer :: errflg + + std_arg = 1 + flag_for_opt_arg = .true. + allocate(opt_arg(nx)) + allocate(opt_arg_2(nx)) + ! capgen does not default-initialize host data; the host must. Zero these + ! so the post-ccpp_init checks below test real wiring, not stale memory. + opt_arg = 0 + opt_arg_2 = 0 + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP register step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_register(suite_name=trim(ccpp_suite), errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_register:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_init(suite_name=trim(ccpp_suite), errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_init:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + ! std_arg must all be 1, opt_arg must all be 0 + write(output_unit, '(a)') "After ccpp_init: check std_arg(:)==1, opt_arg(:)==0, opt_arg_2(:)==0" + if (.not. all(std_arg == 1)) write(error_unit, '(a,3i3)') "Error after ccpp_init: std_arg=", std_arg + if (.not. all(opt_arg == 0)) write(error_unit, '(a,3i3)') "Error after ccpp_init: opt_arg=", opt_arg + if (.not. all(opt_arg_2 == 0)) write(error_unit, '(a,3es13.5)') "Error after ccpp_init: opt_arg_2=", opt_arg_2 + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_physics_init(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_init:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + ! std_arg must all be 1, opt_arg must all be 0 + write(output_unit, '(a)') "PASS: After ccpp_physics_init: check std_arg(:)==1 and opt_arg(:)==0" + if (.not. all(std_arg == 1)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_init: std_arg=", std_arg + if (.not. all(opt_arg == 0)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_init: opt_arg=", opt_arg + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics timestep init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_physics_timestep_init(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + ! std_arg must all be 1, opt_arg must all be 2 + write(output_unit, '(a)') "PASS: After ccpp_physics_timestep_init: check std_arg(:)==1 and opt_arg(:)==2" + if (.not. all(std_arg == 1)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_timestep_init: std_arg=", std_arg + if (.not. all(opt_arg == 2)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_timestep_init: opt_arg=", opt_arg + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics run step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_physics_run(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_run:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + ! std_arg must all be 1, opt_arg must all be 3 + write(output_unit, '(a)') "PASS: After ccpp_physics_run: check std_arg(:)==1 and opt_arg(:)==3" + if (.not. all(std_arg == 1)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_run: std_arg=", std_arg + if (.not. all(opt_arg == 3)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_run: opt_arg=", opt_arg + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics timestep finalize step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + deallocate(opt_arg) + flag_for_opt_arg = .false. + + call ccpp_physics_timestep_final(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_final:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + ! std_arg must all be 7, opt_arg no longer allocated + write(output_unit, '(a)') "PASS: After ccpp_physics_timestep_final: check std_arg(:)==7; opt_arg not allocated" + if (.not. all(std_arg == 7)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_timestep_final: std_arg=", std_arg + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics finalize step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_physics_final(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_final:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + ! std_arg must all be 7, opt_arg no longer allocated + write(output_unit, '(a)') "PASS: After ccpp_physics_final: check std_arg(:)==7; opt_arg not allocated" + if (.not. all(std_arg == 7)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_final: std_arg=", std_arg + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP finalize step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_final(suite_name=trim(ccpp_suite), errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_final:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + +end program test_opt_arg diff --git a/end-to-end-tests/opt_arg/main.meta b/end-to-end-tests/opt_arg/main.meta new file mode 100644 index 00000000..b90de81e --- /dev/null +++ b/end-to-end-tests/opt_arg/main.meta @@ -0,0 +1,65 @@ +[ccpp-table-properties] + name = main + type = control + dependencies = + +[ccpp-arg-table] + name = main + type = control +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer diff --git a/test_prebuild/test_opt_arg/opt_arg_scheme.F90 b/end-to-end-tests/opt_arg/opt_arg_scheme.F90 similarity index 91% rename from test_prebuild/test_opt_arg/opt_arg_scheme.F90 rename to end-to-end-tests/opt_arg/opt_arg_scheme.F90 index 33be0973..963b5264 100644 --- a/test_prebuild/test_opt_arg/opt_arg_scheme.F90 +++ b/end-to-end-tests/opt_arg/opt_arg_scheme.F90 @@ -12,7 +12,7 @@ module opt_arg_scheme private public :: opt_arg_scheme_timestep_init, & opt_arg_scheme_run, & - opt_arg_scheme_timestep_finalize + opt_arg_scheme_timestep_final contains @@ -62,10 +62,10 @@ subroutine opt_arg_scheme_run(nx, var, opt_var, opt_var_2, errmsg, errflg) end if end subroutine opt_arg_scheme_run - !! \section arg_table_opt_arg_scheme_timestep_finalize Argument Table + !! \section arg_table_opt_arg_scheme_timestep_final Argument Table !! \htmlinclude opt_arg_scheme_timestep_finalize.html !! - subroutine opt_arg_scheme_timestep_finalize(nx, var, opt_var, opt_var_2, errmsg, errflg) + subroutine opt_arg_scheme_timestep_final(nx, var, opt_var, opt_var_2, errmsg, errflg) character(len=*), intent(out) :: errmsg integer, intent(out) :: errflg integer, intent(in) :: nx @@ -85,6 +85,6 @@ subroutine opt_arg_scheme_timestep_finalize(nx, var, opt_var, opt_var_2, errmsg, if (present(opt_var_2)) then opt_var_2 = opt_var_2 + 5.0_kind_phys end if - end subroutine opt_arg_scheme_timestep_finalize + end subroutine opt_arg_scheme_timestep_final end module opt_arg_scheme diff --git a/test_prebuild/test_opt_arg/opt_arg_scheme.meta b/end-to-end-tests/opt_arg/opt_arg_scheme.meta similarity index 98% rename from test_prebuild/test_opt_arg/opt_arg_scheme.meta rename to end-to-end-tests/opt_arg/opt_arg_scheme.meta index a00519ec..c0c9a4bf 100644 --- a/test_prebuild/test_opt_arg/opt_arg_scheme.meta +++ b/end-to-end-tests/opt_arg/opt_arg_scheme.meta @@ -107,7 +107,7 @@ ######################################################################## [ccpp-arg-table] - name = opt_arg_scheme_timestep_finalize + name = opt_arg_scheme_timestep_final type = scheme [errmsg] standard_name = ccpp_error_message diff --git a/test_prebuild/test_opt_arg/suite_opt_arg_suite.xml b/end-to-end-tests/opt_arg/suite_opt_arg_suite.xml similarity index 79% rename from test_prebuild/test_opt_arg/suite_opt_arg_suite.xml rename to end-to-end-tests/opt_arg/suite_opt_arg_suite.xml index e66514a4..b91ba5e7 100644 --- a/test_prebuild/test_opt_arg/suite_opt_arg_suite.xml +++ b/end-to-end-tests/opt_arg/suite_opt_arg_suite.xml @@ -1,6 +1,6 @@ - + opt_arg_scheme diff --git a/end-to-end-tests/suite_allocate/CMakeLists.txt b/end-to-end-tests/suite_allocate/CMakeLists.txt new file mode 100644 index 00000000..3c887fb7 --- /dev/null +++ b/end-to-end-tests/suite_allocate/CMakeLists.txt @@ -0,0 +1,69 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "make_workspace" "use_workspace") +set(HOST_FILES "data" "main") +set(SUITE_FILES "suite_allocate_suite.xml") +set(HOST "test_host") +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") + +# Run ccpp_capgen +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +add_executable(test_suite_allocate.x + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES_FILTERED} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES} +) +target_link_libraries(test_suite_allocate.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_suite_allocate.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_suite_allocate.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_suite_allocate + COMMAND test_suite_allocate.x) diff --git a/end-to-end-tests/suite_allocate/README.md b/end-to-end-tests/suite_allocate/README.md new file mode 100644 index 00000000..b1601d11 --- /dev/null +++ b/end-to-end-tests/suite_allocate/README.md @@ -0,0 +1,33 @@ +# suite_allocate test + +Covers the one suite-data feature the rest of the tests do not: a +suite-owned, scheme-allocated variable (allocatable = True). + +scratch_workspace_field is produced by make_workspace (intent=out, +allocatable = True) and consumed by use_workspace. No host table declares +it, so capgen promotes it to a suite-owned variable stored in +ccpp__data. + +Crucially, its dimension workspace_dimension is also suite-owned: it is set +by use_workspace in the timestep_init phase (which runs after +ccpp_init/suite_data_init_fields) — even though use_workspace is listed +after make_workspace in the suite. Phases run suite-wide in order, so the +dimension set in timestep_init is available to every scheme's run. Because +the size is unknown at init, init_fields cannot allocate the array; the +producing scheme must, in the run phase. A non-allocatable version of this is +exactly what validate_init_dimensions rejects. Because it is allocatable: + +- suite_data_init_fields must skip its allocation (the scheme owns it), +- the producing scheme allocates the suite-data component at run time, +- the whole allocated component is passed to the (non-allocatable) consumer dummy, +- suite_data_final_fields frees it (guarded; suite owns teardown). + +The driver asserts the consumer's reduction (workspace_checksum == nw*(nw+1)/2) +and a clean error code. Built with -fcheck=all, so any double-free or +use-after-free in the scheme-allocates / suite-frees ownership split fails the +test. + +This is distinct from: +- capgen — suite-owned vars allocated at init from a register-set dim + (non-allocatable path), and a host-owned allocatable var (model_times). +- nested_suite / var_compat — standalone DDTs with module_name. diff --git a/end-to-end-tests/suite_allocate/data.F90 b/end-to-end-tests/suite_allocate/data.F90 new file mode 100644 index 00000000..fed06847 --- /dev/null +++ b/end-to-end-tests/suite_allocate/data.F90 @@ -0,0 +1,17 @@ +module data + + !! \section arg_table_data Argument Table + !! \htmlinclude data.html + !! + use ccpp_kinds, only: kind_phys + + implicit none + + private + + public :: checksum + + ! Host-owned scalar the consuming scheme fills from the suite-owned workspace. + real(kind=kind_phys) :: checksum + +end module data diff --git a/end-to-end-tests/suite_allocate/data.meta b/end-to-end-tests/suite_allocate/data.meta new file mode 100644 index 00000000..aa051e82 --- /dev/null +++ b/end-to-end-tests/suite_allocate/data.meta @@ -0,0 +1,14 @@ +[ccpp-table-properties] + name = data + type = host + dependencies = +[ccpp-arg-table] + name = data + type = host +[checksum] + standard_name = workspace_checksum + long_name = sum of the suite-owned scratch workspace + units = 1 + dimensions = () + type = real + kind = kind_phys diff --git a/end-to-end-tests/suite_allocate/main.F90 b/end-to-end-tests/suite_allocate/main.F90 new file mode 100644 index 00000000..c2afd946 --- /dev/null +++ b/end-to-end-tests/suite_allocate/main.F90 @@ -0,0 +1,114 @@ +program test_suite_allocate + + use, intrinsic :: iso_fortran_env, only: output_unit, & + error_unit + + use ccpp_kinds, only: kind_phys + + use data, only: checksum + + use test_host_ccpp_cap, only: ccpp_register, & + ccpp_init, & + ccpp_physics_init, & + ccpp_physics_timestep_init, & + ccpp_physics_run, & + ccpp_physics_timestep_final, & + ccpp_physics_final, & + ccpp_final + + implicit none + + character(len=*), parameter :: ccpp_suite = 'suite_allocate_suite' + character(len=512) :: errmsg + integer :: errflg + ! Must match the value use_workspace sets in its timestep_init phase. + integer, parameter :: expected_size = 4 + real(kind=kind_phys) :: expected + real(kind=kind_phys), parameter :: tol = 1.0e-6_kind_phys + + ! use_workspace sets workspace_dimension = expected_size in timestep_init; + ! make_workspace fills work(i) = i in run, so the consumer's sum is N*(N+1)/2. + expected = real(expected_size * (expected_size + 1) / 2, kind_phys) + checksum = -1.0_kind_phys + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP register step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_register(suite_name=trim(ccpp_suite), errmsg=errmsg, errflg=errflg) + call check_err('ccpp_register', errflg, errmsg) + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP init step (suite_data_init_fields runs; ! + ! it must SKIP the allocatable suite var) ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_init(suite_name=trim(ccpp_suite), errmsg=errmsg, errflg=errflg) + call check_err('ccpp_init', errflg, errmsg) + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics init / timestep init steps ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_physics_init(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, & + suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + call check_err('ccpp_physics_init', errflg, errmsg) + + call ccpp_physics_timestep_init(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, & + suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + call check_err('ccpp_physics_timestep_init', errflg, errmsg) + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics run step: producer allocates the ! + ! suite-owned workspace, consumer reduces it ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_physics_run(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, & + suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + call check_err('ccpp_physics_run', errflg, errmsg) + + if (abs(checksum - expected) > tol) then + write(error_unit, '(a,f0.6,a,f0.6)') & + "Error after ccpp_physics_run: workspace_checksum=", checksum, & + " expected ", expected + stop 1 + end if + write(output_unit, '(a,f0.6)') & + "PASS: After ccpp_physics_run: suite-owned allocatable workspace summed to ", checksum + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics timestep final / final steps ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_physics_timestep_final(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, & + suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + call check_err('ccpp_physics_timestep_final', errflg, errmsg) + + call ccpp_physics_final(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, & + suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + call check_err('ccpp_physics_final', errflg, errmsg) + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP finalize step (final_fields frees the ! + ! suite-owned allocatable var; guarded) ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_final(suite_name=trim(ccpp_suite), errmsg=errmsg, errflg=errflg) + call check_err('ccpp_final', errflg, errmsg) + + write(output_unit, '(a)') "PASS: suite_allocate test completed" + +contains + + subroutine check_err(phase, errflg, errmsg) + character(len=*), intent(in) :: phase + integer, intent(in) :: errflg + character(len=*), intent(in) :: errmsg + if (errflg /= 0) then + write(error_unit, '(a)') "An error occurred in " // trim(phase) // ":" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + end subroutine check_err + +end program test_suite_allocate diff --git a/end-to-end-tests/suite_allocate/main.meta b/end-to-end-tests/suite_allocate/main.meta new file mode 100644 index 00000000..b90de81e --- /dev/null +++ b/end-to-end-tests/suite_allocate/main.meta @@ -0,0 +1,65 @@ +[ccpp-table-properties] + name = main + type = control + dependencies = + +[ccpp-arg-table] + name = main + type = control +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer diff --git a/end-to-end-tests/suite_allocate/make_workspace.F90 b/end-to-end-tests/suite_allocate/make_workspace.F90 new file mode 100644 index 00000000..bf1d3c68 --- /dev/null +++ b/end-to-end-tests/suite_allocate/make_workspace.F90 @@ -0,0 +1,41 @@ +!>\file make_workspace.F90 +!! Producer scheme: allocates and fills a suite-owned scratch workspace. +!! The workspace standard name is not provided by the host, so capgen +!! promotes it to a suite-owned, scheme-allocated (allocatable=True) variable +!! stored in ccpp__data. suite_data_init_fields skips its allocation; +!! this scheme owns it; final_fields frees it. + +module make_workspace + + use ccpp_kinds, only: kind_phys + + implicit none + + private + public :: make_workspace_run + +contains + + !! \section arg_table_make_workspace_run Argument Table + !! \htmlinclude make_workspace_run.html + !! + subroutine make_workspace_run(nw, work, errmsg, errflg) + integer, intent(in) :: nw + real(kind=kind_phys), allocatable, intent(out) :: work(:) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + + integer :: i + + errmsg = '' + errflg = 0 + + ! intent(out) on an allocatable dummy auto-deallocates on entry, so this is + ! safe to call repeatedly (the persistent suite-data component is reset here). + allocate(work(nw)) + do i = 1, nw + work(i) = real(i, kind_phys) + end do + end subroutine make_workspace_run + +end module make_workspace diff --git a/test_prebuild/test_track_variables/scheme_A.meta b/end-to-end-tests/suite_allocate/make_workspace.meta similarity index 54% rename from test_prebuild/test_track_variables/scheme_A.meta rename to end-to-end-tests/suite_allocate/make_workspace.meta index 4fc6118c..39dde2a7 100644 --- a/test_prebuild/test_track_variables/scheme_A.meta +++ b/end-to-end-tests/suite_allocate/make_workspace.meta @@ -1,19 +1,27 @@ -######################################################################## [ccpp-table-properties] - name = scheme_A + name = make_workspace type = scheme + dependencies = -######################################################################## [ccpp-arg-table] - name = scheme_A_run + name = make_workspace_run type = scheme -[im] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent +[nw] + standard_name = workspace_dimension + long_name = size of the scratch workspace units = count dimensions = () type = integer intent = in +[work] + standard_name = scratch_workspace_field + long_name = suite-owned scratch workspace allocated by the producing scheme + units = 1 + dimensions = (workspace_dimension) + type = real + kind = kind_phys + intent = out + allocatable = True [errmsg] standard_name = ccpp_error_message long_name = error message for error handling in CCPP diff --git a/end-to-end-tests/suite_allocate/suite_allocate_suite.xml b/end-to-end-tests/suite_allocate/suite_allocate_suite.xml new file mode 100644 index 00000000..4f3bc223 --- /dev/null +++ b/end-to-end-tests/suite_allocate/suite_allocate_suite.xml @@ -0,0 +1,8 @@ + + + + + make_workspace + use_workspace + + diff --git a/end-to-end-tests/suite_allocate/use_workspace.F90 b/end-to-end-tests/suite_allocate/use_workspace.F90 new file mode 100644 index 00000000..c2d66e13 --- /dev/null +++ b/end-to-end-tests/suite_allocate/use_workspace.F90 @@ -0,0 +1,52 @@ +!>\file use_workspace.F90 +!! Consumer scheme: reads the suite-owned scratch workspace allocated by +!! make_workspace and reduces it into a host-owned scalar. Receiving the +!! suite-owned allocatable component through a plain (non-allocatable) dummy +!! exercises capgen passing the whole allocated component to a consumer. + +module use_workspace + + use ccpp_kinds, only: kind_phys + + implicit none + + private + public :: use_workspace_timestep_init, use_workspace_run + +contains + + !! \section arg_table_use_workspace_timestep_init Argument Table + !! \htmlinclude use_workspace_timestep_init.html + !! + subroutine use_workspace_timestep_init(nw, errmsg, errflg) + integer, intent(out) :: nw + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + + errmsg = '' + errflg = 0 + + ! Own the suite workspace dimension here, in timestep_init -- a phase that + ! runs AFTER ccpp_init/suite_data_init_fields. make_workspace (listed earlier + ! but executed in the later run phase) allocates work(nw) using this value, + ! which is exactly why scratch_workspace_field must be allocatable=True: + ! init_fields cannot size it. + nw = 4 + end subroutine use_workspace_timestep_init + + !! \section arg_table_use_workspace_run Argument Table + !! \htmlinclude use_workspace_run.html + !! + subroutine use_workspace_run(work, checksum, errmsg, errflg) + real(kind=kind_phys), intent(in) :: work(:) + real(kind=kind_phys), intent(out) :: checksum + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + + errmsg = '' + errflg = 0 + + checksum = sum(work) + end subroutine use_workspace_run + +end module use_workspace diff --git a/end-to-end-tests/suite_allocate/use_workspace.meta b/end-to-end-tests/suite_allocate/use_workspace.meta new file mode 100644 index 00000000..daa9270a --- /dev/null +++ b/end-to-end-tests/suite_allocate/use_workspace.meta @@ -0,0 +1,65 @@ +[ccpp-table-properties] + name = use_workspace + type = scheme + dependencies = + +[ccpp-arg-table] + name = use_workspace_timestep_init + type = scheme +[nw] + standard_name = workspace_dimension + long_name = size of the scratch workspace + units = count + dimensions = () + type = integer + intent = out +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = use_workspace_run + type = scheme +[work] + standard_name = scratch_workspace_field + long_name = suite-owned scratch workspace allocated by the producing scheme + units = 1 + dimensions = (workspace_dimension) + type = real + kind = kind_phys + intent = in +[checksum] + standard_name = workspace_checksum + long_name = sum of the suite-owned scratch workspace + units = 1 + dimensions = () + type = real + kind = kind_phys + intent = out +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/test/utils/test_utils.F90 b/end-to-end-tests/utils/test_utils.F90 similarity index 88% rename from test/utils/test_utils.F90 rename to end-to-end-tests/utils/test_utils.F90 index 3ae8d549..425a0c61 100644 --- a/test/utils/test_utils.F90 +++ b/end-to-end-tests/utils/test_utils.F90 @@ -23,6 +23,18 @@ logical function check_list(test_list, chk_list, list_desc, suite_name) check_list = .true. errmsg = '' + ! DH* + write(0,*) '' + do lindex = 1, size(chk_list) + write(errmsg, '(a,i4,1x,a)') 'Debug: chk_list', lindex, trim(chk_list(lindex)) + write(0,*) trim(errmsg) + end do + do lindex = 1, size(test_list) + write(errmsg, '(a,i4,1x,a)') 'Debug: test_list', lindex, trim(test_list(lindex)) + write(0,*) trim(errmsg) + end do + ! *DH + ! Check the list size num_items = size(chk_list) if (size(test_list) /= num_items) then diff --git a/end-to-end-tests/var_compat/CMakeLists.txt b/end-to-end-tests/var_compat/CMakeLists.txt new file mode 100644 index 00000000..18ef84c6 --- /dev/null +++ b/end-to-end-tests/var_compat/CMakeLists.txt @@ -0,0 +1,74 @@ + +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ +set(SCHEME_FILES "effr_calc" "effrs_calc" "effr_diag" "effr_pre" "effr_post" "rad_lw" "rad_sw") +set(HOST_FILES "module_rad_ddt" "test_host_data" "test_host_mod" "test_host") +set(SUITE_FILES "var_compatibility_suite.xml") +set(HOST "test_host") +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") + +# Run ccpp_capgen +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +set(EXTRA_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/../utils/test_utils.F90 +) + +add_executable(test_var_compat.x + ${EXTRA_FILES} + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES_FILTERED} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES} + test_var_compatibility_integration.F90 +) +target_link_libraries(test_var_compat.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_var_compat.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_var_compat.x PROPERTIES LINKER_LANGUAGE Fortran) + +add_test(NAME test_var_compat + COMMAND test_var_compat.x) diff --git a/test/var_compatibility_test/README.md b/end-to-end-tests/var_compat/README.md similarity index 100% rename from test/var_compatibility_test/README.md rename to end-to-end-tests/var_compat/README.md diff --git a/test/var_compatibility_test/effr_calc.F90 b/end-to-end-tests/var_compat/effr_calc.F90 similarity index 100% rename from test/var_compatibility_test/effr_calc.F90 rename to end-to-end-tests/var_compat/effr_calc.F90 diff --git a/test/var_compatibility_test/effr_calc.meta b/end-to-end-tests/var_compat/effr_calc.meta similarity index 87% rename from test/var_compatibility_test/effr_calc.meta rename to end-to-end-tests/var_compat/effr_calc.meta index c3733f13..6361eac6 100644 --- a/test/var_compatibility_test/effr_calc.meta +++ b/end-to-end-tests/var_compat/effr_calc.meta @@ -33,7 +33,7 @@ name = effr_calc_run type = scheme [ ncol ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension type = integer units = count dimensions = () @@ -48,7 +48,7 @@ standard_name = effective_radius_of_stratiform_cloud_rain_particle long_name = effective radius of cloud rain particle in micrometer units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = in @@ -57,7 +57,7 @@ standard_name = effective_radius_of_stratiform_cloud_graupel long_name = effective radius of cloud graupel in micrometer units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = in @@ -66,7 +66,7 @@ standard_name = cloud_graupel_number_concentration long_name = number concentration of cloud graupel units = kg-1 - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = in @@ -75,7 +75,7 @@ standard_name = cloud_ice_number_concentration long_name = number concentration of cloud ice units = kg-1 - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = out @@ -84,7 +84,7 @@ standard_name = effective_radius_of_stratiform_cloud_liquid_water_particle long_name = effective radius of cloud liquid water particle in micrometer units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = inout @@ -92,7 +92,7 @@ standard_name = effective_radius_of_stratiform_cloud_ice_particle long_name = effective radius of cloud ice water particle in micrometer units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = out @@ -101,7 +101,7 @@ standard_name = effective_radius_of_stratiform_cloud_snow_particle long_name = effective radius of cloud snow particle in micrometer units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = 8 intent = inout @@ -110,7 +110,7 @@ standard_name = cloud_liquid_number_concentration long_name = number concentration of cloud liquid units = kg-1 - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = out diff --git a/test/var_compatibility_test/effr_diag.F90 b/end-to-end-tests/var_compat/effr_diag.F90 similarity index 100% rename from test/var_compatibility_test/effr_diag.F90 rename to end-to-end-tests/var_compat/effr_diag.F90 diff --git a/test/nested_suite_test/effr_diag.meta b/end-to-end-tests/var_compat/effr_diag.meta similarity index 96% rename from test/nested_suite_test/effr_diag.meta rename to end-to-end-tests/var_compat/effr_diag.meta index 9e0e4fc2..5a5c9e67 100644 --- a/test/nested_suite_test/effr_diag.meta +++ b/end-to-end-tests/var_compat/effr_diag.meta @@ -36,7 +36,7 @@ standard_name = effective_radius_of_stratiform_cloud_rain_particle long_name = effective radius of cloud rain particle in micrometer units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = in diff --git a/test/var_compatibility_test/effr_post.F90 b/end-to-end-tests/var_compat/effr_post.F90 similarity index 100% rename from test/var_compatibility_test/effr_post.F90 rename to end-to-end-tests/var_compat/effr_post.F90 diff --git a/test/var_compatibility_test/effr_post.meta b/end-to-end-tests/var_compat/effr_post.meta similarity index 96% rename from test/var_compatibility_test/effr_post.meta rename to end-to-end-tests/var_compat/effr_post.meta index 721582a6..703b5ebc 100644 --- a/test/var_compatibility_test/effr_post.meta +++ b/end-to-end-tests/var_compat/effr_post.meta @@ -36,7 +36,7 @@ standard_name = effective_radius_of_stratiform_cloud_rain_particle long_name = effective radius of cloud rain particle in micrometer units = m - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = inout diff --git a/test/var_compatibility_test/effr_pre.F90 b/end-to-end-tests/var_compat/effr_pre.F90 similarity index 100% rename from test/var_compatibility_test/effr_pre.F90 rename to end-to-end-tests/var_compat/effr_pre.F90 diff --git a/test/var_compatibility_test/effr_pre.meta b/end-to-end-tests/var_compat/effr_pre.meta similarity index 96% rename from test/var_compatibility_test/effr_pre.meta rename to end-to-end-tests/var_compat/effr_pre.meta index 251b4175..c47d1abf 100644 --- a/test/var_compatibility_test/effr_pre.meta +++ b/end-to-end-tests/var_compat/effr_pre.meta @@ -37,7 +37,7 @@ standard_name = effective_radius_of_stratiform_cloud_rain_particle long_name = effective radius of cloud rain particle in micrometer units = m - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys intent = inout diff --git a/test/var_compatibility_test/effrs_calc.F90 b/end-to-end-tests/var_compat/effrs_calc.F90 similarity index 100% rename from test/var_compatibility_test/effrs_calc.F90 rename to end-to-end-tests/var_compat/effrs_calc.F90 diff --git a/test/var_compatibility_test/effrs_calc.meta b/end-to-end-tests/var_compat/effrs_calc.meta similarity index 88% rename from test/var_compatibility_test/effrs_calc.meta rename to end-to-end-tests/var_compat/effrs_calc.meta index 9ce7b88e..e2fd1de9 100644 --- a/test/var_compatibility_test/effrs_calc.meta +++ b/end-to-end-tests/var_compat/effrs_calc.meta @@ -9,7 +9,7 @@ standard_name = effective_radius_of_stratiform_cloud_snow_particle units = m type = real | kind = kind_phys - dimensions = (horizontal_loop_extent,vertical_layer_dimension) + dimensions = (horizontal_dimension,vertical_layer_dimension) intent = inout [ errmsg ] standard_name = ccpp_error_message diff --git a/test/var_compatibility_test/module_rad_ddt.F90 b/end-to-end-tests/var_compat/module_rad_ddt.F90 similarity index 100% rename from test/var_compatibility_test/module_rad_ddt.F90 rename to end-to-end-tests/var_compat/module_rad_ddt.F90 diff --git a/test/var_compatibility_test/module_rad_ddt.meta b/end-to-end-tests/var_compat/module_rad_ddt.meta similarity index 100% rename from test/var_compatibility_test/module_rad_ddt.meta rename to end-to-end-tests/var_compat/module_rad_ddt.meta diff --git a/test/var_compatibility_test/rad_lw.F90 b/end-to-end-tests/var_compat/rad_lw.F90 similarity index 100% rename from test/var_compatibility_test/rad_lw.F90 rename to end-to-end-tests/var_compat/rad_lw.F90 diff --git a/test/var_compatibility_test/rad_lw.meta b/end-to-end-tests/var_compat/rad_lw.meta similarity index 89% rename from test/var_compatibility_test/rad_lw.meta rename to end-to-end-tests/var_compat/rad_lw.meta index 883edf1b..bfab7426 100644 --- a/test/var_compatibility_test/rad_lw.meta +++ b/end-to-end-tests/var_compat/rad_lw.meta @@ -6,7 +6,7 @@ name = rad_lw_run type = scheme [ ncol ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension type = integer units = count dimensions = () @@ -15,7 +15,7 @@ standard_name = longwave_radiation_fluxes long_name = longwave radiation fluxes units = W m-2 - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) type = ty_rad_lw intent = inout [ errmsg ] diff --git a/test/var_compatibility_test/rad_sw.F90 b/end-to-end-tests/var_compat/rad_sw.F90 similarity index 100% rename from test/var_compatibility_test/rad_sw.F90 rename to end-to-end-tests/var_compat/rad_sw.F90 diff --git a/test/nested_suite_test/rad_sw.meta b/end-to-end-tests/var_compat/rad_sw.meta similarity index 87% rename from test/nested_suite_test/rad_sw.meta rename to end-to-end-tests/var_compat/rad_sw.meta index d88b9acc..af88530f 100644 --- a/test/nested_suite_test/rad_sw.meta +++ b/end-to-end-tests/var_compat/rad_sw.meta @@ -5,7 +5,7 @@ name = rad_sw_run type = scheme [ ncol ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension type = integer units = count dimensions = () @@ -13,14 +13,14 @@ [ sfc_up_sw ] standard_name = surface_upwelling_shortwave_radiation_flux units = W m2 - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) type = real kind = kind_phys intent = inout [ sfc_down_sw ] standard_name = surface_downwelling_shortwave_radiation_flux units = W m2 - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) type = real kind = kind_phys intent = inout diff --git a/test/nested_suite_test/test_host.F90 b/end-to-end-tests/var_compat/test_host.F90 similarity index 69% rename from test/nested_suite_test/test_host.F90 rename to end-to-end-tests/var_compat/test_host.F90 index 67c7a1ac..d88c1d24 100644 --- a/test/nested_suite_test/test_host.F90 +++ b/end-to-end-tests/var_compat/test_host.F90 @@ -104,11 +104,14 @@ end function check_suite subroutine test_host(retval, test_suites) use test_host_mod, only: ncols - use test_host_ccpp_cap, only: test_host_ccpp_physics_initialize - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_initial - use test_host_ccpp_cap, only: test_host_ccpp_physics_run - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_final - use test_host_ccpp_cap, only: test_host_ccpp_physics_finalize + use test_host_ccpp_cap, only: ccpp_register + use test_host_ccpp_cap, only: ccpp_init + use test_host_ccpp_cap, only: ccpp_physics_init + use test_host_ccpp_cap, only: ccpp_physics_timestep_init + use test_host_ccpp_cap, only: ccpp_physics_run + use test_host_ccpp_cap, only: ccpp_physics_timestep_final + use test_host_ccpp_cap, only: ccpp_physics_final + use test_host_ccpp_cap, only: ccpp_final use test_host_ccpp_cap, only: ccpp_physics_suite_list use test_host_mod, only: init_data, & compare_data @@ -155,10 +158,35 @@ subroutine test_host(retval, test_suites) return end if + ! Register CCPP + do sind = 1, num_suites + call ccpp_register( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in initialize of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + end if + end do + + ! Initialize CCPP + do sind = 1, num_suites + call ccpp_init( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in initialize of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + end if + end do + ! Use the suite information to setup the run do sind = 1, num_suites - call test_host_ccpp_physics_initialize(test_suites(sind)%suite_name, & - errmsg, errflg) + call ccpp_physics_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) if (errflg /= 0) then write(6, '(4a)') 'ERROR in initialize of ', & trim(test_suites(sind)%suite_name), ': ', trim(errmsg) @@ -171,8 +199,11 @@ subroutine test_host(retval, test_suites) exit end if if (errflg == 0) then - call test_host_ccpp_physics_timestep_initial( & - test_suites(sind)%suite_name, errmsg, errflg) + call ccpp_physics_timestep_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) end if if (errflg /= 0) then write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & @@ -199,10 +230,12 @@ subroutine test_host(retval, test_suites) exit end if if (errflg == 0) then - call test_host_ccpp_physics_run( & - test_suites(sind)%suite_name, & - test_suites(sind)%suite_parts(index), & - col_start, col_end, errmsg, errflg) + call ccpp_physics_run( & + suite_name=test_suites(sind)%suite_name, & + group_name=test_suites(sind)%suite_parts(index), & + col_start=col_start, col_end=col_end, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) end if if (errflg /= 0) then write(6, '(5a)') trim(test_suites(sind)%suite_name), & @@ -219,12 +252,37 @@ subroutine test_host(retval, test_suites) exit end if if (errflg == 0) then - call test_host_ccpp_physics_timestep_final( & - test_suites(sind)%suite_name, errmsg, errflg) + call ccpp_physics_timestep_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) end if if (errflg /= 0) then write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_physics_timestep_final, ', & + 'Exiting...' + exit + end if + end do + + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_physics_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + end if + if (errflg /= 0) then + write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_physics_final, ', & + 'Exiting...' exit end if end do @@ -234,13 +292,14 @@ subroutine test_host(retval, test_suites) exit end if if (errflg == 0) then - call test_host_ccpp_physics_finalize( & - test_suites(sind)%suite_name, errmsg, errflg) + call ccpp_final( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) end if if (errflg /= 0) then write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & trim(errmsg) - write(6, '(2a)') 'An error occurred in ccpp_timestep_final, ', & + write(6, '(2a)') 'An error occurred in ccpp_final, ', & 'Exiting...' exit end if diff --git a/end-to-end-tests/var_compat/test_host.meta b/end-to-end-tests/var_compat/test_host.meta new file mode 100644 index 00000000..c151d87b --- /dev/null +++ b/end-to-end-tests/var_compat/test_host.meta @@ -0,0 +1,64 @@ +[ccpp-table-properties] + name = test_host + type = control + +[ccpp-arg-table] + name = test_host + type = control +[ col_start ] + standard_name = horizontal_loop_begin + type = integer + units = count + dimensions = () + protected = True +[ col_end ] + standard_name = horizontal_loop_end + type = integer + units = count + dimensions = () + protected = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer diff --git a/test/var_compatibility_test/test_host_data.F90 b/end-to-end-tests/var_compat/test_host_data.F90 similarity index 100% rename from test/var_compatibility_test/test_host_data.F90 rename to end-to-end-tests/var_compat/test_host_data.F90 diff --git a/test/var_compatibility_test/test_host_data.meta b/end-to-end-tests/var_compat/test_host_data.meta similarity index 94% rename from test/var_compatibility_test/test_host_data.meta rename to end-to-end-tests/var_compat/test_host_data.meta index 59a0fb4d..ec691215 100644 --- a/test/var_compatibility_test/test_host_data.meta +++ b/end-to-end-tests/var_compat/test_host_data.meta @@ -1,3 +1,11 @@ +#[ccpp-table-properties] +# name = test_host_data +# type = host +# dependencies = module_rad_ddt.F90 +#[ccpp-arg-table] +# name = test_host_data +# type = host +# [ccpp-table-properties] name = physics_state type = ddt @@ -42,7 +50,6 @@ dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys - intent = in active = (flag_indicating_cloud_microphysics_has_graupel) [nci] standard_name = cloud_ice_number_concentration @@ -51,7 +58,6 @@ dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys - intent = in active = (flag_indicating_cloud_microphysics_has_ice) [scalar_var] standard_name = scalar_variable_for_testing @@ -118,11 +124,3 @@ units = None dimensions = () type = integer - -[ccpp-table-properties] - name = test_host_data - type = module - dependencies = module_rad_ddt.F90 -[ccpp-arg-table] - name = test_host_data - type = module diff --git a/test/var_compatibility_test/test_host_mod.F90 b/end-to-end-tests/var_compat/test_host_mod.F90 similarity index 99% rename from test/var_compatibility_test/test_host_mod.F90 rename to end-to-end-tests/var_compat/test_host_mod.F90 index d3bde866..cba820d0 100644 --- a/test/var_compatibility_test/test_host_mod.F90 +++ b/end-to-end-tests/var_compat/test_host_mod.F90 @@ -12,7 +12,7 @@ module test_host_mod !! integer, parameter :: ncols = 12 integer, parameter :: pver = 4 - type(physics_state) :: phys_state + type(physics_state), target :: phys_state real(kind=kind_phys) :: effrs(ncols, pver) logical, parameter :: has_ice = .true. logical, parameter :: has_graupel = .true. diff --git a/test/nested_suite_test/test_host_mod.meta b/end-to-end-tests/var_compat/test_host_mod.meta similarity index 97% rename from test/nested_suite_test/test_host_mod.meta rename to end-to-end-tests/var_compat/test_host_mod.meta index 51a2f5c3..a5df9381 100644 --- a/test/nested_suite_test/test_host_mod.meta +++ b/end-to-end-tests/var_compat/test_host_mod.meta @@ -1,9 +1,9 @@ [ccpp-table-properties] name = test_host_mod - type = module + type = host [ccpp-arg-table] name = test_host_mod - type = module + type = host [ ncols] standard_name = horizontal_dimension units = count diff --git a/test/var_compatibility_test/test_var_compatibility_integration.F90 b/end-to-end-tests/var_compat/test_var_compatibility_integration.F90 similarity index 100% rename from test/var_compatibility_test/test_var_compatibility_integration.F90 rename to end-to-end-tests/var_compat/test_var_compatibility_integration.F90 diff --git a/test/var_compatibility_test/var_compatibility_suite.xml b/end-to-end-tests/var_compat/var_compatibility_suite.xml similarity index 100% rename from test/var_compatibility_test/var_compatibility_suite.xml rename to end-to-end-tests/var_compat/var_compatibility_suite.xml diff --git a/test/var_compatibility_test/var_compatibility_test_reports.py b/end-to-end-tests/var_compat/var_compatibility_test_reports.py similarity index 100% rename from test/var_compatibility_test/var_compatibility_test_reports.py rename to end-to-end-tests/var_compat/var_compatibility_test_reports.py diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 59386b60..00000000 --- a/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -addopts = -ra --ignore=scripts/metadata2html.py --ignore-glob=test/**/test_reports.py - diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index be9f272b..00000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -black -flake8 -pytest diff --git a/run_codee_tmp.sh b/run_codee_tmp.sh deleted file mode 100755 index 4483b1ff..00000000 --- a/run_codee_tmp.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -files=( - "src/ccpp_constituent_prop_mod.F90" - "src/ccpp_hashable.F90" - "src/ccpp_hash_table.F90" - "src/ccpp_scheme_utils.F90" - "src/ccpp_types.F90" -) - -for entry in "${files[@]}"; do - file=${entry} - git checkout origin/develop -- $file - codee format --verbose --on-error force $file - echo "" - echo "-------------------------------------------------" -done diff --git a/scripts/ccpp_capgen.py b/scripts/ccpp_capgen.py deleted file mode 100755 index ef512c33..00000000 --- a/scripts/ccpp_capgen.py +++ /dev/null @@ -1,768 +0,0 @@ -#!/usr/bin/env python3 - -""" -Create CCPP parameterization caps, host-model interface code, -physics suite runtime code, and CCPP framework documentation. -""" - -# Python library imports -import sys -import os -import logging -import re -# CCPP framework imports -from ccpp_database_obj import CCPPDatabaseObj -from ccpp_datafile import generate_ccpp_datatable -from ccpp_suite import API -from file_utils import check_for_writeable_file, remove_dir, replace_paths -from file_utils import create_file_list, move_modified_files -from file_utils import KINDS_FILENAME, KINDS_MODULE -from fortran_tools import parse_fortran_file, FortranWriter -from framework_env import parse_command_line -from host_cap import write_host_cap -from host_model import HostModel -from metadata_table import parse_metadata_file, register_ddts, SCHEME_HEADER_TYPE -from parse_tools import init_log, set_log_level, context_string -from parse_tools import register_fortran_ddt_name -from parse_tools import CCPPError, ParseInternalError - -## Capture the Framework root -_SCRIPT_PATH = os.path.dirname(__file__) -_FRAMEWORK_ROOT = os.path.abspath(os.path.join(_SCRIPT_PATH, os.pardir)) -_SRC_ROOT = os.path.join(_FRAMEWORK_ROOT, "src") -## Init this now so that all Exceptions can be trapped -_LOGGER = init_log(os.path.basename(__file__)) - -## Recognized Fortran filename extensions -_FORTRAN_FILENAME_EXTENSIONS = ['F90', 'f90', 'F', 'f'] - -## Metadata table types which can have extra variables in Fortran -_EXTRA_VARIABLE_TABLE_TYPES = ['module', 'host', 'ddt'] - -## Metadata table types where order is significant -_ORDERED_TABLE_TYPES = [] - -## CCPP Framework supported DDT types -_CCPP_FRAMEWORK_DDT_TYPES = ["ccpp_hash_table_t", - "ccpp_hashable_t", - "ccpp_hashable_char_t"] - -############################################################################### -def delete_pathnames_from_file(capfile, logger): -############################################################################### - """Remove all the filenames found in , then delete """ - root_path = os.path.dirname(os.path.abspath(capfile)) - success = True - with open(capfile, 'r') as infile: - for line in infile.readlines(): - path = line.strip() - # Skip blank lines and lines which appear to start with a comment. - if path and (path[0] != '#') and (path[0] != '!'): - # Check for an absolute path - if not os.path.isabs(path): - # Assume relative pathnames are relative to pathsfile - path = os.path.normpath(os.path.join(root_path, path)) - # end if - logger.info("Clean: Removing {}".format(path)) - try: - os.remove(path) - except OSError as oserr: - success = False - errmsg = 'Unable to remove {}\n{}' - logger.warning(errmsg.format(path, oserr)) - # end try - # end if (else skip blank or comment line) - # end for - # end with open - logger.info("Clean: Removing {}".format(capfile)) - try: - os.remove(capfile) - except OSError as oserr: - success = False - errmsg = 'Unable to remove {}\n{}' - logger.warning(errmsg.format(capfile, oserr)) - # end try - if success: - logger.info("ccpp_capgen clean successful, exiting") - else: - logger.info("ccpp_capgen clean encountered errors, exiting") - # end if - -############################################################################### -def find_associated_fortran_file(filename, fortran_source_path): -############################################################################### - """Find the Fortran file associated with metadata file, . - Fortran files should be in . - """ - fort_filename = None - lastdot = filename.rfind('.') - if lastdot < 0: - base = os.path.basename(filename + '.') - else: - base = os.path.basename(filename[0:lastdot+1]) - # end if - for extension in _FORTRAN_FILENAME_EXTENSIONS: - test_name = os.path.join(fortran_source_path, base + extension) - if os.path.exists(test_name): - fort_filename = test_name - break - # end if - # end for - if fort_filename is None: - emsg = f"Cannot find Fortran file associated with '{filename}'." - emsg += f"\nfortran_src_path = '{fortran_source_path}'" - raise CCPPError(emsg) - # end if - return fort_filename - -############################################################################### -def create_kinds_file(run_env, output_dir): -############################################################################### - "Create the kinds.F90 file to be used by CCPP schemes and suites" - kinds_filepath = os.path.join(output_dir, KINDS_FILENAME) - if run_env.logger is not None: - msg = 'Writing {} to {}' - run_env.logger.info(msg.format(KINDS_FILENAME, output_dir)) - # end if - kind_types = run_env.kind_types() - with FortranWriter(kinds_filepath, "w", - "kinds for CCPP", KINDS_MODULE) as kindf: - for kind_type in kind_types: - kind_spec = run_env.kind_spec(kind_type) - use_stmt = f"use {run_env.kind_module(kind_type)}," - if kind_spec == kind_type: - use_stmt += f" only: {kind_type}" - else: - use_stmt += f" only: {kind_type} => {kind_spec}" - # end if - kindf.write(use_stmt, 1) - # end for - kindf.write_preamble() - for kind_type in kind_types: - kindf.write("public :: {}".format(kind_type), 1) - # end for - # end with - return kinds_filepath - -############################################################################### -def add_error(error_string, new_error): -############################################################################### - '''Add an error () to , separating errors by a - newline''' - if error_string: - error_string += '\n' - # end if - return error_string + new_error - -############################################################################### -def is_arrayspec(local_name): -############################################################################### - "Return True iff is an array reference" - return '(' in local_name - -############################################################################### -def find_var_in_list(local_name, var_list): -############################################################################### - """Find a variable, , in . - local name is used because Fortran metadata variables do not have - real standard names. - Note: The search is case insensitive. - Return both the variable and the index where it was found. - If not found, return None for the variable and -1 for the index - """ - vvar = None - vind = -1 - lname = local_name.lower() - for lind, lvar in enumerate(var_list): - if lvar.get_prop_value('local_name').lower() == lname: - vvar = lvar - vind = lind - break - # end if - # end for - return vvar, vind - -############################################################################### -def var_comp(prop_name, mvar, fvar, title, case_sensitive=False): -############################################################################### - "Compare a property between two variables" - errors = '' - mprop = mvar.get_prop_value(prop_name) - fprop = fvar.get_prop_value(prop_name) - if not case_sensitive: - if isinstance(mprop, str): - mprop = mprop.lower() - # end if - if isinstance(fprop, str): - fprop = fprop.lower() - # end if - # end if - comp = mprop == fprop - if not comp: - errmsg = '{} mismatch ({} != {}) in {}{}' - ctx = context_string(mvar.context) - errors = add_error(errors, - errmsg.format(prop_name, mprop, fprop, title, ctx)) - # end if - return errors - -############################################################################### -def dims_comp(mheader, mvar, fvar, title, logger, case_sensitive=False): -############################################################################### - "Compare the dimensions attribute of two variables" - errors = '' - mdims = mvar.get_dimensions() - fdims = mheader.convert_dims_to_standard_names(fvar, logger=logger) - comp = len(mdims) == len(fdims) - if not comp: - errmsg = 'Error: rank mismatch in {}/{} ({} != {}){}' - stdname = mvar.get_prop_value('standard_name') - ctx = context_string(mvar.context) - errors = add_error(errors, errmsg.format(title, stdname, - len(mdims), len(fdims), ctx)) - # end if - if comp: - # Now, compare the dims - for dim_ind, mdim in enumerate(mdims): - if ':' in mdim: - mdim = ':'.join([x.strip() for x in mdim.split(':')]) - # end if - fdim = fdims[dim_ind].strip() - if ':' in fdim: - fdim = ':'.join([x.strip() for x in fdim.split(':')]) - # end if - if not case_sensitive: - mdim = mdim.lower() - fdim = fdim.lower() - # end if - # Naked colon is okay for Fortran side - comp = fdim in (':', fdim) - if not comp: - errmsg = 'Error: dim {} mismatch ({} != {}) in {}/{}{}' - stdname = mvar.get_prop_value('standard_name') - ctx = context_string(mvar.context) - errmsg = errmsg.format(dim_ind+1, mdim, fdims[dim_ind], - title, stdname, ctx) - errors = add_error(errors, errmsg) - # end if - # end for - # end if - return errors - -############################################################################### -def compare_fheader_to_mheader(meta_header, fort_header, logger): -############################################################################### - """Compare a metadata header against the header generated from the - corresponding code in the associated Fortran file. - Return a string with any errors found (empty string is no errors). - """ - errors_found = '' - title = meta_header.title - mht = meta_header.header_type - fht = fort_header.header_type - if mht != fht: - # Special case, host metadata can be in a Fortran module or scheme - if (mht != 'host') or (fht not in ('module', SCHEME_HEADER_TYPE)): - errmsg = 'Metadata table type mismatch for {}, {} != {}{}' - ctx = meta_header.start_context() - raise CCPPError(errmsg.format(title, meta_header.header_type, - fort_header.header_type, ctx)) - # end if - else: - # The headers should have the same variables in the same order - # The exception is that a Fortran module can have variable declarations - # after all the metadata variables. - mlist = meta_header.variable_list() - mlen = len(mlist) - flist = fort_header.variable_list() - flen = len(flist) - # Remove array references from mlist before checking lengths - for mvar in mlist: - if is_arrayspec(mvar.get_prop_value('local_name')): - mlen -= 1 - # end if - # end for - list_match = mlen == flen - # Check for optional Fortran variables that are not in metadata - if flen > mlen: - for find, fvar in enumerate(flist): - lname = fvar.get_prop_value('local_name') - _, mind = find_var_in_list(lname, mlist) - if mind < 0: - if fvar.get_prop_value('optional'): - # This is an optional variable - flen -= 1 - # end if - # end if - # end for - list_match = mlen == flen - # end if - if not list_match: - if fht in _EXTRA_VARIABLE_TABLE_TYPES: - if flen > mlen: - list_match = True - else: - etype = 'Fortran {}'.format(fht) - # end if - elif flen > mlen: - etype = 'metadata header' - else: - etype = 'Fortran {}'.format(fht) - # end if - # end if - if not list_match: - errmsg = 'Variable mismatch in {}, variables missing from {}.' - errors_found = add_error(errors_found, errmsg.format(title, etype)) - if etype == "metadata header": - # Look for missing metadata variables - for fvar in flist: - lname = fvar.get_prop_value('local_name') - _, find = find_var_in_list(lname, mlist) - if (find < 0) and (not fvar.get_prop_value('optional')): - errmsg = f"Fortran variable, {lname}, not in metadata" - errors_found = add_error(errors_found, errmsg) - # end if - # end for - # end if - # end if - for mind, mvar in enumerate(mlist): - lname = mvar.get_prop_value('local_name') - mname = mvar.get_prop_value('standard_name') - arrayref = is_arrayspec(lname) - fvar, find = find_var_in_list(lname, flist) - # Check for consistency between optional variables in metadata and - # optional variables in fortran. Error if optional attribute is - # missing from fortran declaration. - # first check: if metadata says the variable is optional, does the fortran match? - mopt = mvar.get_prop_value('optional') - if find and mopt: - fopt = fvar.get_prop_value('optional') - if (not fopt): - errmsg = f'Missing "optional" attribute in fortran declaration for variable {mname}, ' \ - f'for {title}' - errors_found = add_error(errors_found, errmsg) - # end if - # end if - # now check: if fortran says the variable is optional, does the metadata match? - if fvar: - fopt = fvar.get_prop_value('optional') - if (fopt and not mopt): - errmsg = f'Missing "optional" metadata property for variable {mname}, ' \ - f'for {title}' - errors_found = add_error(errors_found, errmsg) - # end if - # end if - if mind >= flen: - if arrayref: - # Array reference, variable not in Fortran table - pass - elif fvar is None: - errmsg = 'No Fortran variable for {} in {}' - errors_found = add_error(errors_found, - errmsg.format(lname, title)) - # end if (no else, we already reported an out-of-place error - # Do not break to collect all missing variables - continue - # end if - # At this point, we should have a Fortran variable - if (not arrayref) and (fvar is None): - errmsg = 'Variable mismatch in {}, no Fortran variable {}.' - errors_found = add_error(errors_found, errmsg.format(title, - lname)) - continue - # end if - # Check order dependence - if fht in _ORDERED_TABLE_TYPES: - if find != mind: - errmsg = 'Out of order argument, {} in {}' - errors_found = add_error(errors_found, - errmsg.format(lname, title)) - continue - # end if - # end if - if arrayref: - # Array reference, do not look for this in Fortran table - continue - # end if - errs = var_comp('local_name', mvar, fvar, title) - if errs: - errors_found = add_error(errors_found, errs) - else: - errs = var_comp('type', mvar, fvar, title) - if errs: - errors_found = add_error(errors_found, errs) - # end if - errs = var_comp('kind', mvar, fvar, title) - if errs: - errors_found = add_error(errors_found, errs) - # end if - if meta_header.header_type == SCHEME_HEADER_TYPE: - errs = var_comp('intent', mvar, fvar, title) - if errs: - errors_found = add_error(errors_found, errs) - # end if - # end if - # Compare dimensions - errs = dims_comp(meta_header, mvar, fvar, title, logger) - if errs: - errors_found = add_error(errors_found, errs) - # end if - # end if - # end for - # end if - return errors_found - -############################################################################### -def check_fortran_against_metadata(meta_headers, fort_headers, - mfilename, ffilename, logger, - fortran_routines=None): -############################################################################### - """Compare a set of metadata headers from against the - code in the associated Fortran file, . - NB: This routine destroys the list, but returns the - contents in an association dictionary on successful completion.""" - header_dict = {} # Associate a Fortran header for every metadata header - for mheader in meta_headers: - fheader = None - mtitle = mheader.title - for findex in range(len(fort_headers)): #pylint: disable=consider-using-enumerate - if fort_headers[findex].title == mtitle: - fheader = fort_headers.pop(findex) - break - # end if - # end for - if fheader is None: - tlist = '\n '.join([x.title for x in fort_headers]) - logger.debug("CCPP routines in {}:{}".format(ffilename, tlist)) - errmsg = "No matching Fortran routine found for {} in {}" - raise CCPPError(errmsg.format(mtitle, ffilename)) - # end if - header_dict[mheader] = fheader - # end if - # end while - if fort_headers: - errmsgs = [] - estr = "No matching metadata header found for {} in {}" - for fheader in fort_headers: - if fheader.has_variables: - errmsgs.append(estr.format(fheader.title, mfilename)) - # end if - # end for - if errmsgs: - mheads = ', '.join([x.name for x in meta_headers]) - errmsgs.append(f'Metadata headers in file: {mheads}') - raise CCPPError('\n'.join(errmsgs)) - # end if - # end if - # We have a one-to-one set, compare headers - errors_found = '' - for mheader in header_dict: - fheader = header_dict[mheader] - errors_found += compare_fheader_to_mheader(mheader, fheader, logger) - # end for - if errors_found: - num_errors = len(re.findall(r'\n', errors_found)) + 1 - errmsg = "{}\n{} error{} found comparing {} to {}" - raise CCPPError(errmsg.format(errors_found, num_errors, - 's' if num_errors > 1 else '', - mfilename, ffilename)) - # end if - # No return, an exception is raised on error - -############################################################################### -def duplicate_item_error(title, filename, itype, orig_item): -############################################################################### - """Raise an error indicating a duplicate item of type, """ - errmsg = "Duplicate {typ}, {title}, found in {file}" - edict = {'title':title, 'file':filename, 'typ':itype} - ofile = orig_item.context.filename - if ofile is not None: - errmsg = errmsg + ", original found in {ofile}" - edict['ofile'] = ofile - # end if - raise CCPPError(errmsg.format(**edict)) - -############################################################################### -def parse_host_model_files(host_filenames, host_name, run_env, - known_ddts=list()): -############################################################################### - """ - Gather information from host files (e.g., DDTs, registry) and - return a host model object with the information. - """ - header_dict = {} - table_dict = {} - logger = run_env.logger - for filename in host_filenames: - logger.info('Reading host model data from {}'.format(filename)) - # parse metadata file - mtables = parse_metadata_file(filename, known_ddts, run_env) - fortran_source_path = mtables[0].fortran_source_path - fort_file = find_associated_fortran_file(filename, fortran_source_path) - ftables, _ = parse_fortran_file(fort_file, run_env) - # Check Fortran against metadata (will raise an exception on error) - mheaders = list() - for sect in [x.sections() for x in mtables]: - mheaders.extend(sect) - # end for - fheaders = list() - for sect in [x.sections() for x in ftables]: - fheaders.extend(sect) - # end for - check_fortran_against_metadata(mheaders, fheaders, - filename, fort_file, logger) - # Check for duplicate tables, then add to dict - for table in mtables: - if table.table_name in table_dict: - duplicate_item_error(table.table_name, filename, - table.table_type, table_dict[header.title]) - else: - table_dict[table.table_name] = table - # end if - # end for - # Check for duplicate headers, then add to dict - for header in mheaders: - if header.title in header_dict: - duplicate_item_error(header.title, filename, - header.header_type, - header_dict[header.title]) - else: - header_dict[header.title] = header - if header.header_type == 'ddt': - known_ddts.append(header.title) - # end if - # end for - # end for - if not host_name: - host_name = None - # end if - host_model = HostModel(table_dict, host_name, run_env) - return host_model - -############################################################################### -def parse_scheme_files(scheme_filenames, run_env, skip_ddt_check=False, - known_ddts=list(), relative_source_path=False): -############################################################################### - """ - Gather information from scheme files (e.g., init, run, and finalize - methods) and return resulting dictionary. - """ - table_dict = {} # Duplicate check and for dependencies processing - header_dict = {} # To check for duplicates - logger = run_env.logger - for filename in scheme_filenames: - logger.info('Reading CCPP schemes from {}'.format(filename)) - # parse metadata file - mtables = parse_metadata_file(filename, known_ddts, run_env, - skip_ddt_check=skip_ddt_check, - relative_source_path=relative_source_path) - fortran_source_path = mtables[0].fortran_source_path - fort_file = find_associated_fortran_file(filename, fortran_source_path) - ftables, additional_routines = parse_fortran_file(fort_file, run_env) - # Check Fortran against metadata (will raise an exception on error) - mheaders = list() - for sect in [x.sections() for x in mtables]: - mheaders.extend(sect) - # end for - fheaders = list() - for sect in [x.sections() for x in ftables]: - fheaders.extend(sect) - # end for - check_fortran_against_metadata(mheaders, fheaders, - filename, fort_file, logger, - fortran_routines=additional_routines) - # Check for duplicate tables, then add to dict - for table in mtables: - if table.table_name in table_dict: - duplicate_item_error(table.table_name, filename, - table.table_type, table_dict[header.title]) - else: - table_dict[table.table_name] = table - # end if - # end for - # Check for duplicate headers, then add to dict - for header in mheaders: - if header.title in header_dict: - duplicate_item_error(header.title, filename, header.header_type, - header_dict[header.title]) - else: - header_dict[header.title] = header - if header.header_type == 'ddt': - known_ddts.append(header.title) - # end if - # end if - # end for - # end for - - return header_dict.values(), table_dict - -############################################################################### -def clean_capgen(cap_output_file, logger): -############################################################################### - """Attempt to remove the files created by the last invocation of capgen""" - log_level = logger.getEffectiveLevel() - set_log_level(logger, logging.INFO) - if os.path.exists(cap_output_file): - logger.info("Cleaning capgen files from {}".format(cap_output_file)) - delete_pathnames_from_file(cap_output_file, logger) - else: - emsg = "Unable to run clean, {} not found" - logger.error(emsg.format(cap_output_file)) - # end if - set_log_level(logger, log_level) - -############################################################################### -def capgen(run_env, return_db=False): -############################################################################### - """Parse indicated host, scheme, and suite files. - Generate code to allow host model to run indicated CCPP suites.""" - ## A few sanity checks - ## Make sure output directory is legit - if os.path.exists(run_env.output_dir): - if not os.path.isdir(run_env.output_dir): - errmsg = "output-root, '{}', is not a directory" - raise CCPPError(errmsg.format(run_env.output_root)) - # end if - if not os.access(run_env.output_dir, os.W_OK): - errmsg = "Cannot write files to output-root ({})" - raise CCPPError(errmsg.format(run_env.output_root)) - # end if (output_dir is okay) - else: - # Try to create output_dir (let it crash if it fails) - os.makedirs(run_env.output_dir) - # end if - # Pre-register base CCPP DDT types: - for ddt_name in _CCPP_FRAMEWORK_DDT_TYPES: - register_fortran_ddt_name(ddt_name) - # end for - src_dir = os.path.join(_FRAMEWORK_ROOT, "src") - host_files = run_env.host_files - host_name = run_env.host_name - scheme_files = run_env.scheme_files - # We need to create three lists of files, hosts, schemes, and SDFs - host_files = create_file_list(run_env.host_files, ['meta'], 'Host', - run_env.logger) - # The host model needs to know about the constituents module - const_mod = os.path.join(_SRC_ROOT, "ccpp_constituent_prop_mod.meta") - if const_mod not in host_files: - host_files.append(const_mod) - # end if - scheme_files = create_file_list(run_env.scheme_files, ['meta'], - 'Scheme', run_env.logger) - sdfs = create_file_list(run_env.suites, ['xml'], 'Suite', run_env.logger) - check_for_writeable_file(run_env.datatable_file, "Cap output datatable") - ##XXgoldyXX: Temporary warning - if run_env.generate_docfiles: - raise CCPPError("--generate-docfiles not yet supported") - # end if - # The host model may depend on suite DDTs - scheme_ddts = register_ddts(scheme_files) - # Handle the host files - host_model = parse_host_model_files(host_files, host_name, run_env, - known_ddts=scheme_ddts) - # Next, parse the scheme files - # We always need to parse the constituent DDTs - const_prop_mod = os.path.join(src_dir, "ccpp_constituent_prop_mod.meta") - if const_prop_mod not in scheme_files: - scheme_files = [const_prop_mod] + scheme_files - # end if - host_ddts = register_ddts(host_files) - scheme_headers, scheme_tdict = parse_scheme_files(scheme_files, run_env, - known_ddts=host_ddts) - if run_env.verbose: - ddts = host_model.ddt_lib.keys() - if ddts: - run_env.logger.debug("DDT definitions = {}".format(ddts)) - # end if - # end if - plist = host_model.prop_list('local_name') - if run_env.verbose: - run_env.logger.debug("{} variables = {}".format(host_model.name, plist)) - run_env.logger.debug("schemes = {}".format([x.title - for x in scheme_headers])) - # Finally, we can get on with writing suites - # Make sure to write to temporary location if files exist in - if not os.path.exists(run_env.output_dir): - # Try to create output_dir (let it crash if it fails) - os.makedirs(run_env.output_dir) - # Nothing here, use it for output - outtemp_dir = run_env.output_dir - elif not os.listdir(run_env.output_dir): - # Nothing here, use it for output - outtemp_dir = run_env.output_dir - else: - # We need to create a temporary staging area, create it here - outtemp_name = "ccpp_temp_scratch_dir" - outtemp_dir = os.path.join(run_env.output_dir, outtemp_name) - if os.path.exists(outtemp_dir): - remove_dir(outtemp_dir, force=True) - # end if - os.makedirs(outtemp_dir) - # end if - ccpp_api = API(sdfs, host_model, scheme_headers, run_env) - cap_filenames = ccpp_api.write(outtemp_dir, run_env) - if run_env.generate_host_cap: - # Create a cap file - cap_module = host_model.ccpp_cap_name() - host_files = [write_host_cap(host_model, ccpp_api, cap_module, - outtemp_dir, run_env)] - else: - host_files = list() - # end if - # Create the kinds file - kinds_file = create_kinds_file(run_env, outtemp_dir) - # Move any changed files to output_dir and remove outtemp_dir - move_modified_files(outtemp_dir, run_env.output_dir, - overwrite=run_env.force_overwrite, remove_src=True) - # We have to rename the files we created - if outtemp_dir != run_env.output_dir: - replace_paths(cap_filenames, outtemp_dir, run_env.output_dir) - replace_paths(host_files, outtemp_dir, run_env.output_dir) - kinds_file = kinds_file.replace(outtemp_dir, run_env.output_dir) - # end if - # Finally, create the database of generated files and caps - # This can be directly in output_dir because it will not affect dependencies - generate_ccpp_datatable(run_env, host_model, ccpp_api, - scheme_headers, scheme_tdict, host_files, - cap_filenames, kinds_file, src_dir) - if return_db: - return CCPPDatabaseObj(run_env, host_model=host_model, api=ccpp_api) - # end if - return None - -############################################################################### -def _main_func(): -############################################################################### - """Parse command line, then parse indicated host, scheme, and suite files. - Finally, generate code to allow host model to run indicated CCPP suites.""" - framework_env = parse_command_line(sys.argv[1:], __doc__, logger=_LOGGER) - if framework_env.verbosity > 1: - set_log_level(framework_env.logger, logging.DEBUG) - elif framework_env.verbosity > 0: - set_log_level(framework_env.logger, logging.INFO) - # end if - if framework_env.clean: - clean_capgen(framework_env.datatable_file, framework_env.logger) - else: - _ = capgen(framework_env) - # end if (clean) - -############################################################################### - -if __name__ == "__main__": - try: - _main_func() - sys.exit(0) - except ParseInternalError as pie: - _LOGGER.exception(pie) - sys.exit(-1) - except CCPPError as ccpp_err: - if _LOGGER.getEffectiveLevel() <= logging.DEBUG: - _LOGGER.exception(ccpp_err) - else: - _LOGGER.error(ccpp_err) - # end if - sys.exit(1) - finally: - logging.shutdown() - # end try diff --git a/scripts/ccpp_database_obj.py b/scripts/ccpp_database_obj.py deleted file mode 100644 index 24579750..00000000 --- a/scripts/ccpp_database_obj.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python3 - -""" -Define the CCPPDatabaseObj object -Object definition and methods to provide information from a run of capgen. -""" - -from host_model import HostModel -from ccpp_suite import API - -class CCPPDatabaseObjError(ValueError): - """Error class specific to CCPPDatabaseObj. - All uses of this error should be internal (i.e., programmer error, - not user error).""" - - def __init__(self, message): - """Initialize this exception""" - super().__init__(message) - -class CCPPDatabaseObj: - """Object with data and methods to provide information from a run of capgen. - """ - - def __init__(self, run_env, host_model=None, api=None, database_file=None): - """Initialize this CCPPDatabaseObj. - If is not None, all other inputs MUST be None and - the object is created from the database table created by capgen. - To initialize the object from an in-memory capgen run, ALL other - inputs MUST be passed (i.e., not None) and it is an error to pass - a value for . - """ - - runtime_obj = all([host_model is not None, api is not None]) - self.__host_model = None - self.__api = None - self.__database_file = None - if runtime_obj and database_file: - emsg = "Cannot provide both runtime arguments and database_file." - elif (not runtime_obj) and (not database_file): - emsg = "Must provide either database_file or all runtime arguments." - else: - emsg = "" - # end if - if emsg: - raise CCPPDatabaseObjError(f"ERROR: {emsg}") - # end if - if runtime_obj: - self.__host_model = host_model - self.__api = api - else: - self.db_from_file(run_env, database_file) - # end if - - def db_from_file(self, run_env, database_file): - """Create the necessary internal data structures from a CCPP - datatable.xml file created by capgen. - """ - metadata_tables = {} - host_name = "host" - self.__host_model = HostModel(metadata_tables, host_name, run_env) - self.__api = API(sdfs, host_model, scheme_headers, run_env) - raise CCPPDatabaseObjError("ERROR: not supported") - - def host_model_dict(self): - """Return the host model dictionary for this CCPP DB object""" - if self.__host_model is not None: - return self.__host_model - # end if - raise CCPPDatabaseObjError("ERROR: not supported") - - def suite_list(self): - """Return a list of suites built into the API""" - if self.__api is not None: - return list(self.__api.suites) - # end if - raise CCPPDatabaseObjError("ERROR: not supported") - - def constituent_dictionary(self, suite): - """Return the constituent dictionary for """ - return suite.constituent_dictionary() - - def call_list(self, phase): - """Return the API call list for """ - if self.__api is not None: - return self.__api.call_list(phase) - # end if - raise CCPPDatabaseObjError("ERROR: not supported") diff --git a/scripts/ccpp_fortran_to_metadata.py b/scripts/ccpp_fortran_to_metadata.py deleted file mode 100755 index 56d466e4..00000000 --- a/scripts/ccpp_fortran_to_metadata.py +++ /dev/null @@ -1,258 +0,0 @@ -#!/usr/bin/env python3 - -#pylint: disable=anomalous-backslash-in-string -""" -Create prototype CCPP metadata tables from Fortran files - -Parses annotated Fortran files to produce metadata files where the -standard_name, units, and dimension standard names must be filled in. -The annotation is a two line comment for every physics scheme, derived -data type (DDT) definition, or host model data section. -The annotation form is: - -!> \section arg_table_ Argument Table -!! \htmlinclude arg_table_.html - -where is the name of the scheme, the name of the DDT, or the -name of the module containing data to be included in the metadata file. -For a scheme, the annotation must appear just before the subroutine statement. -For a DDT definition, the annotation must appear just before the type statement. -For module data, the annotation should occur after any module variables - which should not be included in the metadata file. -Note that only CCPP interfaces (e.g., _run, _init, _final) - will be documented in this manner. All other routines should be left as is. -""" -#pylint: enable=anomalous-backslash-in-string - -# Python library imports -import argparse -import sys -import os -import os.path -import logging -# CCPP framework imports -from framework_env import CCPPFrameworkEnv -from parse_tools import init_log, set_log_level -from parse_tools import CCPPError, ParseInternalError -from parse_tools import reset_standard_name_counter, unique_standard_name -from parse_tools import register_fortran_ddt_name -from fortran_tools import parse_fortran_file -from file_utils import create_file_list -from metadata_table import blank_metadata_line - -## Init this now so that all Exceptions can be trapped -_LOGGER = init_log(os.path.basename(__file__)) - -## Recognized Fortran filename extensions -_FORTRAN_FILENAME_EXTENSIONS = ['F90', 'f90', 'F', 'f'] - -############################################################################### -def parse_command_line(args, description): -############################################################################### - "Create an ArgumentParser to parse and return command-line arguments" - parser = argparse.ArgumentParser(description=description, - formatter_class=argparse.RawTextHelpFormatter) - - parser.add_argument("files", metavar='', - type=str, - help="""Comma separated list of filenames to process -Filenames with a '.meta' suffix are treated as host model metadata files -Filenames with a '.txt' suffix are treated as containing a list of .meta -filenames""") - - parser.add_argument("--preproc-directives", - metavar='VARDEF1[,VARDEF2 ...]', type=str, default='', - help="Proprocessor directives used to correctly parse source files") - - parser.add_argument("--ddt-names", - metavar='DDT_NAME1[,DDT_NAME2 ...]', type=str, default='', - help="Comma-separated DDT names that may be used in the parsed Fortran files") - - parser.add_argument("--output-root", type=str, - metavar='', - default=os.getcwd(), - help="directory for generated files") - - parser.add_argument("--section-separator", type=str, default='', - help="""Comment line to separate CCPP metadata tables -(must start with a # or ; character)""") - - parser.add_argument("--verbose", action='count', default=0, - help="Log more activity, repeat for increased output") - pargs = parser.parse_args(args) - return pargs - -############################################################################### -def write_metadata_file(mfilename, ftables, sep): -############################################################################### - """ - Write the prototype metadata file, , based on the - headers () parsed from Fortran. - """ - # Write the metadata file with all the items collected from Fortran - with open(mfilename, 'w') as outfile: - header_sep = '' - table_name = '' - for table in ftables: - # Write the table properties section - # Note that there may be extra tables depending on how the - # Fortran was parsed. - if (not table_name) or (table_name != table.table_name): - outfile.write("{}[ccpp-table-properties]{}".format(header_sep, - os.linesep)) - header_sep = sep + os.linesep - table_name = table.table_name - outfile.write(" name = {}{}".format(table_name, os.linesep)) - outfile.write(" type = {}{}".format(table.table_type, - os.linesep)) - # end if - for header in table.sections(): - lname_dict = {'1':'ccpp_constant_one'} - outfile.write('{}[ccpp-arg-table]{}'.format(header_sep, - os.linesep)) - outfile.write(' name = {}{}'.format(header.title, - os.linesep)) - outfile.write(' type = {}{}'.format(header.header_type, - os.linesep)) - for var in header.variable_list(): - lname = var.get_prop_value('local_name') - outfile.write('[ {} ]{}'.format(lname, os.linesep)) - prop = var.get_prop_value('standard_name') - outfile.write(' standard_name = {}{}'.format(prop, - os.linesep)) - lname_dict[lname] = prop - prop = var.get_prop_value('units') - if not prop: - prop = 'enter_units' - # end if - outfile.write(' units = {}{}'.format(prop, os.linesep)) - tprop = var.get_prop_value('type') - kprop = var.get_prop_value('kind') - if tprop == kprop: - outfile.write(' type = {}'.format(tprop)) - else: - outfile.write(' type = {}'.format(tprop.lower())) - if kprop: - outfile.write(' | kind = {}'.format(kprop.lower())) - # end if - # end if - outfile.write(os.linesep) - dims = var.get_dimensions() - # Fill in standard names for dimensions - dlist = list() - if dims: - for dim in dims: - dslist = list() - for dimspec in dim.split(':'): - if dimspec and (dimspec in lname_dict): - dstr = lname_dict[dimspec] - else: - dstr = unique_standard_name() - # end if - dslist.append(dstr) - # end for - dlist.append(':'.join(dslist)) - # end for - # end if - prop = '(' + ','.join(dlist) + ')' - outfile.write(' dimensions = {}{}'.format(prop, - os.linesep)) - if header.header_type == 'scheme': - prop = var.get_prop_value('intent') - outfile.write(' intent = {}{}'.format(prop, - os.linesep)) - # end if - # end for - # end for - # end for - # end with - -############################################################################### -def parse_fortran_files(filenames, run_env, output_dir, sep, logger): -############################################################################### - """ - Parse each file in and produce a prototype metadata file - with a metadata table for each arg_table entry in the file. - """ - meta_filenames = list() - for filename in filenames: - logger.info('Looking for arg_tables from {}'.format(filename)) - reset_standard_name_counter() - ftables, _ = parse_fortran_file(filename, run_env) - # Create metadata filename - filepath = '.'.join(os.path.basename(filename).split('.')[0:-1]) - fname = filepath + '.meta' - mfilename = os.path.join(output_dir, fname) - write_metadata_file(mfilename, ftables, sep) - meta_filenames.append(mfilename) - return meta_filenames - -############################################################################### -def _main_func(): -############################################################################### - """Parse command line, then parse indicated Fortran files. - Finally, generate a prototype metadata file for each Fortran file.""" - args = parse_command_line(sys.argv[1:], __doc__) - verbosity = args.verbose - if verbosity > 1: - set_log_level(_LOGGER, logging.DEBUG) - elif verbosity > 0: - set_log_level(_LOGGER, logging.INFO) - # end if - if args.ddt_names: - for dname in args.ddt_names.split(','): - register_fortran_ddt_name(dname) - # end for - # end if - # Make sure we know where output is going - output_dir = os.path.abspath(args.output_root) - # Optional table separator comment - section_sep = args.section_separator - if not blank_metadata_line(section_sep): - emsg = "Illegal section separator, '{}', first character must be # or ;" - raise CCPPError(emsg.format(section_sep)) - # We need to create a list of input Fortran files - fort_files = create_file_list(args.files, _FORTRAN_FILENAME_EXTENSIONS, - 'Fortran', _LOGGER) - preproc_defs = args.preproc_directives - ## A few sanity checks - ## Make sure output directory is legit - if os.path.exists(output_dir): - if not os.path.isdir(output_dir): - errmsg = "output-root, '{}', is not a directory" - raise CCPPError(errmsg.format(args.output_root)) - # end if - if not os.access(output_dir, os.W_OK): - errmsg = "Cannot write files to output-root ({})" - raise CCPPError(errmsg.format(args.output_root)) - # end if (output_dir is okay) - else: - # Try to create output_dir (let it crash if it fails) - os.makedirs(output_dir) - # end if - # Parse the files and create metadata - run_env = CCPPFrameworkEnv(_LOGGER, verbose=verbosity, - host_files="", scheme_files="", suites="", - preproc_directives=preproc_defs) - _ = parse_fortran_files(fort_files, run_env, - output_dir, section_sep, _LOGGER) - -############################################################################### - -if __name__ == "__main__": - try: - _main_func() - sys.exit(0) - except ParseInternalError as pie: - _LOGGER.exception(pie) - sys.exit(-1) - except CCPPError as ccpp_err: - if _LOGGER.getEffectiveLevel() <= logging.DEBUG: - _LOGGER.exception(ccpp_err) - else: - _LOGGER.error(ccpp_err) - # end if - sys.exit(1) - finally: - logging.shutdown() - # end try diff --git a/scripts/ccpp_prebuild.py b/scripts/ccpp_prebuild.py deleted file mode 100755 index c6198e27..00000000 --- a/scripts/ccpp_prebuild.py +++ /dev/null @@ -1,856 +0,0 @@ -#!/usr/bin/env python3 - -# Standard modules -import argparse -import collections -import copy -import filecmp -import importlib -import itertools -import logging -import os -import re -import sys - -# CCPP framework imports -from common import lowercase_keys_and_values -from common import encode_container, decode_container, decode_container_as_dict -from common import CCPP_STAGES, CCPP_INTERNAL_VARIABLES, CCPP_STATIC_API_MODULE, CCPP_INTERNAL_VARIABLE_DEFINITON_FILE -from common import STANDARD_VARIABLE_TYPES, STANDARD_INTEGER_TYPE, CCPP_TYPE -from common import SUITE_DEFINITION_FILENAME_PATTERN -from common import split_var_name_and_array_reference -from metadata_parser import merge_dictionaries, parse_scheme_tables, parse_variable_tables -from mkcap import CapsMakefile, CapsCMakefile, CapsSourcefile, \ - SchemesMakefile, SchemesCMakefile, SchemesSourcefile, \ - TypedefsMakefile, TypedefsCMakefile, TypedefsSourcefile -from mkdoc import metadata_to_html, metadata_to_latex -from mkstatic import API, Suite, Group -from mkstatic import CCPP_SUITE_VARIABLES - -############################################################################### -# Set up the command line argument parser and other global variables # -############################################################################### - -parser = argparse.ArgumentParser() -parser.add_argument('--config', action='store', help='path to CCPP prebuild configuration file', required=True) -parser.add_argument('--clean', action='store_true', help='remove files created by this script, then exit', default=False) -parser.add_argument('--verbose', action='store_true', help='enable verbose output from this script', default=False) -parser.add_argument('--debug', action='store_true', help='enable debugging features in auto-generated code', default=False) -parser.add_argument('--suites', action='store', help='suite definition files to use (comma-separated, without path)', default='') -parser.add_argument('--builddir', action='store', help='relative path to CCPP build directory', required=False, default=None) -parser.add_argument('--namespace', action='store', help='namespace suffix to be added to the name of static api module', required=False, default='') - -# BASEDIR is the current directory where this script is executed -BASEDIR = os.getcwd() - -############################################################################### -# Functions and subroutines # -############################################################################### - -def parse_arguments(): - """Parse command line arguments.""" - success = True - args = parser.parse_args() - configfile = args.config - clean = args.clean - verbose = args.verbose - debug = args.debug - if args.suites: - sdfs = ['{0}.xml'.format(x) for x in args.suites.split(',')] - else: - sdfs = None - builddir = args.builddir - namespace = args.namespace - return (success, configfile, clean, verbose, debug, sdfs, builddir, namespace) - -def import_config(configfile, builddir): - """Import the configuration from a given configuration file""" - success = True - config = {} - - if not os.path.isfile(configfile): - logging.error("Configuration file {0} not found".format(configfile)) - success = False - return(success, config) - - # Import the host-model specific CCPP prebuild config; - # split into path and module name for import - configpath = os.path.abspath(os.path.dirname(configfile)) - configmodule = os.path.splitext(os.path.basename(configfile))[0] - sys.path.append(configpath) - ccpp_prebuild_config = importlib.import_module(configmodule) - - # If the build directory for running ccpp_prebuild.py is not - # specified as command line argument, use value from config - if not builddir: - builddir = os.path.join(BASEDIR, ccpp_prebuild_config.DEFAULT_BUILD_DIR) - logging.info('Build directory not specified on command line, ' + \ - 'use "{}" from CCPP prebuild config'.format(ccpp_prebuild_config.DEFAULT_BUILD_DIR)) - - # Definitions in host-model dependent CCPP prebuild config script - config['variable_definition_files'] = ccpp_prebuild_config.VARIABLE_DEFINITION_FILES - config['typedefs_makefile'] = ccpp_prebuild_config.TYPEDEFS_MAKEFILE.format(build_dir=builddir) - config['typedefs_cmakefile'] = ccpp_prebuild_config.TYPEDEFS_CMAKEFILE.format(build_dir=builddir) - config['typedefs_sourcefile'] = ccpp_prebuild_config.TYPEDEFS_SOURCEFILE.format(build_dir=builddir) - config['scheme_files'] = ccpp_prebuild_config.SCHEME_FILES - config['schemes_makefile'] = ccpp_prebuild_config.SCHEMES_MAKEFILE.format(build_dir=builddir) - config['schemes_cmakefile'] = ccpp_prebuild_config.SCHEMES_CMAKEFILE.format(build_dir=builddir) - config['schemes_sourcefile'] = ccpp_prebuild_config.SCHEMES_SOURCEFILE.format(build_dir=builddir) - config['caps_makefile'] = ccpp_prebuild_config.CAPS_MAKEFILE.format(build_dir=builddir) - config['caps_cmakefile'] = ccpp_prebuild_config.CAPS_CMAKEFILE.format(build_dir=builddir) - config['caps_sourcefile'] = ccpp_prebuild_config.CAPS_SOURCEFILE.format(build_dir=builddir) - config['caps_dir'] = ccpp_prebuild_config.CAPS_DIR.format(build_dir=builddir) - config['suites_dir'] = ccpp_prebuild_config.SUITES_DIR.format(build_dir=builddir) - config['host_model'] = ccpp_prebuild_config.HOST_MODEL_IDENTIFIER - config['html_vartable_file'] = ccpp_prebuild_config.HTML_VARTABLE_FILE.format(build_dir=builddir) - config['latex_vartable_file'] = ccpp_prebuild_config.LATEX_VARTABLE_FILE.format(build_dir=builddir) - # Location of static API file, shell script to source, cmake include file - config['static_api_dir'] = ccpp_prebuild_config.STATIC_API_DIR.format(build_dir=builddir) - config['static_api_sourcefile'] = ccpp_prebuild_config.STATIC_API_SOURCEFILE.format(build_dir=builddir) - config['static_api_cmakefile'] = ccpp_prebuild_config.STATIC_API_CMAKEFILE.format(build_dir=builddir) - - # To handle new metadata: import DDT references (if exist) - try: - config['typedefs_new_metadata'] = ccpp_prebuild_config.TYPEDEFS_NEW_METADATA - logging.info("Found TYPEDEFS_NEW_METADATA dictionary in config, assume at least some data is in new metadata format") - except AttributeError: - config['typedefs_new_metadata'] = None - logging.info("Could not find TYPEDEFS_NEW_METADATA dictionary in config, assume all data is in old metadata format") - - return(success, config) - -def setup_logging(verbose): - """Sets up the logging module and logging level.""" - success = True - if verbose: - level = logging.DEBUG - else: - level = logging.INFO - logging.basicConfig(format='%(levelname)s: %(message)s', level=level) - if verbose: - logging.info('Logging level set to DEBUG') - else: - logging.info('Logging level set to INFO') - return success - -def clean_files(config, namespace): - """Clean files created by ccpp_prebuild.py""" - success = True - logging.info('Performing clean ....') - if namespace: - static_api_file = '{api}.F90'.format(api=CCPP_STATIC_API_MODULE+'_'+namespace) - else: - static_api_file = '{api}.F90'.format(api=CCPP_STATIC_API_MODULE) - # Create list of files to remove, use wildcards where necessary - files_to_remove = [ - config['typedefs_makefile'], - config['typedefs_cmakefile'], - config['typedefs_sourcefile'], - config['schemes_makefile'], - config['schemes_cmakefile'], - config['schemes_sourcefile'], - config['caps_makefile'], - config['caps_cmakefile'], - config['caps_sourcefile'], - config['html_vartable_file'], - config['latex_vartable_file'], - os.path.join(config['caps_dir'], 'ccpp_*_cap.F90'), - os.path.join(config['static_api_dir'], static_api_file), - config['static_api_sourcefile'], - ] - for f in files_to_remove: - try: - os.remove(f) - except FileNotFoundError: - pass - except Exception as e: - logging.error(f"Error removing {f}: {e}") - success = False - return success - -def get_all_suites(suites_dir): - """Assemble a list of all suite definition files in suites_dir""" - success = False - logging.info("No suites were given, compiling a list of all suites") - sdfs = [] - for f in os.listdir(suites_dir): - match = SUITE_DEFINITION_FILENAME_PATTERN.match(f) - if match: - logging.info('Adding suite definition file {}'.format(f)) - sdfs.append(f) - if sdfs: - success = True - return (success, sdfs) - -def parse_suites(suites_dir, sdfs): - """Parse suite definition files for prebuild""" - logging.info('Parsing suite definition files ...') - suites = [] - for sdf in sdfs: - sdf_file=os.path.join(suites_dir, sdf) - if not os.path.exists(sdf_file): - # If suite file not found, check old filename convention (suite_[suitename].xml) - sdf_file_legacy=os.path.join(suites_dir, f"suite_{sdf}") - if os.path.exists(sdf_file_legacy): - logging.warning("Parsing suite definition file using legacy naming convention") - logging.warning(f"Filename {os.path.basename(sdf_file_legacy)}") - logging.warning(f"Suite name {sdf}") - sdf_file=sdf_file_legacy - else: - logging.critical(f"Suite definition file {sdf_file} not found.") - success = False - return (success, suites) - - logging.info(f'Parsing suite definition file {sdf_file} ...') - suite = Suite(sdf_name=sdf_file) - success = suite.parse() - if not success: - logging.error('Parsing suite definition file {0} failed.'.format(sdf)) - break - suites.append(suite) - return (success, suites) - -def convert_local_name_from_new_metadata(metadata, standard_name, typedefs_new_metadata, converted_variables): - """Convert local names in new metadata format (no old-style DDT references, array references as - standard names) to old metadata format (with old-style DDT references, array references as local names).""" - success = True - var = metadata[standard_name][0] - # Check if this variable has already been converted - if standard_name in converted_variables: - logging.debug('Variable {0} was in old metadata format and has already been converted'.format(standard_name)) - return (success, var.local_name, converted_variables) - # Decode container into a dictionary - container = decode_container_as_dict(var.container) - # Check if variable is in old or new metadata format - module_name = container['MODULE'] - if not module_name in typedefs_new_metadata.keys(): - logging.debug('Variable {0} is in old metadata format, no conversion necessary'.format(standard_name)) - return (success, var.local_name, converted_variables) - # For module variables set type_name to module_name - if not 'TYPE' in container.keys(): - type_name = module_name - else: - type_name = container['TYPE'] - # Check that this module/type is configured (modules will have empty prefices) - if not type_name in typedefs_new_metadata[module_name].keys(): - logging.error("Module {0} uses the new metadata format, but module/type {1} is not configured".format(module_name, type_name)) - success = False - return (success, None, converted_variables) - - # The local name (incl. the array reference) is in new metadata format - local_name = var.local_name - logging.debug("Converting local name {0} of variable {1} from new to old metadata".format(local_name, standard_name)) - if "(" in local_name: - (actual_var_name, array_reference) = split_var_name_and_array_reference(local_name) - indices = array_reference.lstrip('(').rstrip(')').split(',') - indices_local_names = [] - for index_range in indices: - # Remove leading and trailing whitespaces - index_range = index_range.strip() - # Leave colons-only dimension alone - if index_range == ':': - indices_local_names.append(index_range) - continue - # Split by colons to get a pair of dimensions - dimensions = index_range.split(':') - dimensions_local_names = [] - for dimension in dimensions: - # Remove leading and trailing whitespaces - dimension = dimension.strip() - # Leave literals alone - try: - int(dimension) - dimensions_local_names.append(dimension) - continue - except ValueError: - pass - # Convert the local name of the dimension to old metadata standard, if necessary (recursive call) - (success, local_name_dim, converted_variables) = convert_local_name_from_new_metadata( - metadata, dimension, typedefs_new_metadata, converted_variables) - if not success: - return (success, None, converted_variables) - # Update the local name of the dimension, if necessary - if not metadata[dimension][0].local_name == local_name_dim: - logging.debug("Updating local name of variable {0} from {1} to {2}".format(dimension, - metadata[dimension][0].local_name, local_name_dim)) - metadata[dimension][0].local_name = local_name_dim - dimensions_local_names.append(local_name_dim) - indices_local_names.append(':'.join(dimensions_local_names)) - # Put back together the array reference with local names in old metadata format - array_reference_local_names = '(' + ','.join(indices_local_names) + ')' - # Compose local name (still without any DDT reference prefix) - local_name = actual_var_name + array_reference_local_names - - # Prefix the local name with the reference if not empty - if typedefs_new_metadata[module_name][type_name]: - local_name = typedefs_new_metadata[module_name][type_name] + '%' + local_name - if success: - converted_variables.append(standard_name) - - return (success, local_name, converted_variables) - -def gather_variable_definitions(variable_definition_files, typedefs_new_metadata): - """Scan all Fortran source files with variable definitions on the host model side. - If typedefs_new_metadata is not None, search all metadata entries and convert new metadata - (local names) into old metadata by prepending the DDT references.""" - # - logging.info('Parsing metadata tables for variables provided by host model ...') - success = True - metadata_define = collections.OrderedDict() - dependencies_define = collections.OrderedDict() - for variable_definition_file in variable_definition_files: - (filedir, filename) = os.path.split(os.path.abspath(variable_definition_file)) - # Change to directory of variable_definition_file and parse it - os.chdir(os.path.join(BASEDIR,filedir)) - (metadata, dependencies) = parse_variable_tables(filedir, filename) - metadata_define = merge_dictionaries(metadata_define, metadata) - dependencies_define.update(dependencies) - # Return to BASEDIR - os.chdir(BASEDIR) - # - if typedefs_new_metadata: - logging.info('Convert local names from new metadata format into old metadata format ...') - # Keep track of which variables have already been converted - converted_variables = [] - for key in metadata_define.keys(): - # Double-check that variable definitions are unique - if len(metadata_define[key])>1: - logging.error("Multiple definitions of standard_name {0} in type/variable defintions".format(key)) - success = False - return - (success, local_name, converted_variables) = convert_local_name_from_new_metadata( - metadata_define, key, typedefs_new_metadata, converted_variables) - if not success: - logging.error("An error occurred during the conversion of variable {0} from new to old metadata format".format(key)) - return (success, metadata_define) - # Update the local name of the variable, if necessary - if not metadata_define[key][0].local_name == local_name: - logging.debug("Updating local name of variable {0} from {1} to {2}".format(key, - metadata_define[key][0].local_name, local_name)) - metadata_define[key][0].local_name = local_name - # - return (success, metadata_define, dependencies_define) - -def collect_physics_subroutines(scheme_files): - """Scan all Fortran source files in scheme_files for subroutines with argument tables.""" - logging.info('Parsing metadata tables in physics scheme files ...') - success = True - # Parse all scheme files: record metadata, argument list, dependencies, and which scheme is in which file - metadata_request = collections.OrderedDict() - arguments_request = collections.OrderedDict() - dependencies_request = collections.OrderedDict() - schemes_in_files = collections.OrderedDict() - for scheme_file in scheme_files: - scheme_file_with_abs_path = os.path.abspath(scheme_file) - (scheme_filepath, scheme_filename) = os.path.split(scheme_file_with_abs_path) - # Change to directory where scheme_file lives - os.chdir(scheme_filepath) - (metadata, arguments, dependencies) = parse_scheme_tables(scheme_filepath, scheme_filename) - # Record which scheme is in which file - for scheme in arguments.keys(): - schemes_in_files[scheme] = scheme_file_with_abs_path - # Merge metadata, append to arguments and dependencies - metadata_request = merge_dictionaries(metadata_request, metadata) - arguments_request.update(arguments) - dependencies_request.update(dependencies) - os.chdir(BASEDIR) - # Return to BASEDIR - os.chdir(BASEDIR) - return (success, metadata_request, arguments_request, dependencies_request, schemes_in_files) - -def check_schemes_in_suites(arguments, suites): - """Check that all schemes that are requested in the suites exist""" - success = True - logging.info("Checking for existence of schemes in suites ...") - argument_keys = [x.lower() for x in arguments.keys()] - for suite in suites: - for group in suite.groups: - for subcycle in group.subcycles: - for scheme_name in subcycle.schemes: - if not scheme_name in argument_keys: - success = False - logging.critical("Scheme {} in suite {} cannot be found".format(scheme_name, suite.name)) - return success - -def filter_metadata(metadata, arguments, dependencies, schemes_in_files, suites): - """Remove all variables from metadata that are not used in the given suite; - also remove information on argument lists, dependencies and schemes in files""" - success = True - # Output: filtered dictionaries - metadata_filtered = collections.OrderedDict() - arguments_filtered = collections.OrderedDict() - dependencies_filtered = collections.OrderedDict() - schemes_in_files_filtered = collections.OrderedDict() - # Loop through all variables and check if the calling subroutine is in list of subroutines - for var_name in sorted(metadata.keys()): - keep = False - for var in metadata[var_name][:]: - container_string = decode_container(var.container) - subroutine = container_string[container_string.find('SUBROUTINE')+len('SUBROUTINE')+1:] - # Replace the full CCPP stage name with the abbreviated version - for ccpp_stage in CCPP_STAGES.keys(): - subroutine = subroutine.replace(ccpp_stage, CCPP_STAGES[ccpp_stage]) - for suite in suites: - if subroutine in suite.all_subroutines_called: - keep = True - break - if keep: - break - if keep: - metadata_filtered[var_name] = metadata[var_name] - else: - logging.info("filtering out variable {0}".format(var_name)) - # Filter argument lists - for scheme in arguments.keys(): - for suite in suites: - if scheme.lower() in suite.all_schemes_called: - arguments_filtered[scheme] = arguments[scheme] - break - # Filter dependencies - for scheme in dependencies.keys(): - for suite in suites: - if scheme.lower() in suite.all_schemes_called: - dependencies_filtered[scheme] = dependencies[scheme] - break - # Filter schemes_in_files - for scheme in schemes_in_files.keys(): - for suite in suites: - if scheme.lower() in suite.all_schemes_called: - schemes_in_files_filtered[scheme] = schemes_in_files[scheme] - return (success, metadata_filtered, arguments_filtered, dependencies_filtered, schemes_in_files_filtered) - -def add_ccpp_suite_variables(metadata): - """ Add variables that are required to construct CCPP suites to the list of requested variables""" - success = True - logging.info("Adding CCPP suite variables to list of requested variables") - for var_name in CCPP_SUITE_VARIABLES.keys(): - if not var_name in metadata.keys(): - metadata[var_name] = [copy.deepcopy(CCPP_SUITE_VARIABLES[var_name])] - logging.debug("Adding CCPP suite variable {0} to list of requested variables".format(var_name)) - return (success, metadata) - -def generate_list_of_schemes_and_dependencies_to_compile(schemes_in_files, dependencies1, dependencies2): - """Generate a flat list of schemes and dependencies in two dependency dictionaries to compile""" - success = True - # schemes_in_files is a dictionary with key scheme_name and value scheme_file - # dependencies is a dictionary with key scheme_name and value "list of dependencies" - schemes_and_dependencies_to_compile = list(schemes_in_files.values()) + \ - [dependency for dependency_list in list(dependencies1.values()) for dependency in dependency_list] + \ - [dependency for dependency_list in list(dependencies2.values()) for dependency in dependency_list] - # Remove duplicates - return (success, list(set(schemes_and_dependencies_to_compile))) - -def compare_metadata(metadata_define, metadata_request): - """Compare the requested metadata to the defined one. For each requested entry, a - single (i.e. non-ambiguous entry) must be present in the defined entries.""" - - logging.info('Comparing metadata for requested and provided variables ...') - success = True - modules = [] - metadata = collections.OrderedDict() - for var_name in sorted(metadata_request.keys()): - # Check that variable is provided by the model - if not var_name in metadata_define.keys(): - requested_by = ' & '.join(var.container for var in metadata_request[var_name]) - success = False - logging.error('Variable {0} requested by {1} not provided by the model'.format(var_name, requested_by)) - continue - # Check that an unambiguous target exists for this variable - if len(metadata_define[var_name]) > 1: - success = False - requested_by = ' & '.join(var.container for var in metadata_request[var_name]) - provided_by = ' & '.join(var.container for var in metadata_define[var_name]) - error_message = ' error, variable {0} requested by {1} cannot be identified unambiguously.'.format(var_name, requested_by) +\ - ' Multiple definitions in {0}'.format(provided_by) - logging.error(error_message) - continue - # Check that the variable properties are compatible between the model and the schemes; - # because we know that all variables in the metadata_request[var_name] list are compatible, - # it is sufficient to test the first entry against (the unique) metadata_define[var_name][0]. - if not metadata_request[var_name][0].compatible(metadata_define[var_name][0]): - success = False - error_message = ' incompatible entries in metadata for variable {0}:\n'.format(var_name) +\ - ' provided: {0}\n'.format(metadata_define[var_name][0].print_debug()) +\ - ' requested: {0}'.format(metadata_request[var_name][0].print_debug()) - logging.error(error_message) - continue - # Check for and register unit conversions if necessary. This must be done for each registered - # variable in the metadata_request[var_name] list (i.e. for each subroutine that is using it). - # Because var is an instance of the variable specific to the subroutine that uses it, and since - # each variable can be passed to a subroutine only once, there can be no overlapping/conflicting - # unit conversions. - for var in metadata_request[var_name]: - # Compare units - if var.units == metadata_define[var_name][0].units: - continue - # Register conversion, depending on the intent for this subroutine. - logging.debug('Registering unit conversion for variable {0} in {1}'.format(var_name, var.container)) - if var.intent=='inout': - var.convert_from(metadata_define[var_name][0].units) - var.convert_to(metadata_define[var_name][0].units) - elif var.intent=='in': - var.convert_from(metadata_define[var_name][0].units) - elif var.intent=='out': - var.convert_to(metadata_define[var_name][0].units) - # If the host model variable is allocated based on a condition, i.e. has an active attribute other - # than T (.true.), the scheme variable must be optional - if not metadata_define[var_name][0].active == 'T': - for var in metadata_request[var_name]: - if var.optional == 'F': - # DH 20241022 - change logging.error to logging.warn, because it is known - # that this strict check is not correct and will be reverted soon - #logging.error( - logging.warn("Conditionally allocated host-model variable {0} is not optional in {1}".format( - var_name, var.container)) - #success = False - # TEMPORARY CHECK - IF THE VARIABLE IS ALWAYS ALLOCATED, THE SCHEME VARIABLE SHOULDN'T BE OPTIONAL - else: - for var in metadata_request[var_name]: - if var.optional == 'T': - logging.warn("Unconditionally allocated host-model variable {0} is optional in {1}".format( - var_name, var.container)) - - # Construct the actual target variable and list of modules to use from the information in 'container' - var = metadata_define[var_name][0] - target = '' - for item in var.container.split(' '): - subitems = item.split('_') - if subitems[0] == 'MODULE': - # Add to list of required modules - modules.append('_'.join(subitems[1:])) - elif subitems[0] == 'TYPE': - pass - else: - logging.error('Unknown identifier {0} in container value of defined variable {1}'.format(subitems[0], var_name)) - success = False - target += var.local_name - # Copy the length kind from the variable definition to update len=* in the variable requests - if var.type == 'character': - kind = var.kind - metadata[var_name] = metadata_request[var_name] - # Set target and kind (if applicable) - for var in metadata[var_name]: - var.target = target - logging.debug('Requested variable {0} in {1} matched to target {2} in module {3}'.format( - var_name, var.container, target, modules[-1])) - # Update len=* for character variables - if var.type == 'character' and var.kind == 'len=*': - logging.debug('Update kind information for requested variable {0} in {1} from {2} to {3}'.format(var_name, - var.container, var.kind, kind)) - var.kind = kind - - # Remove duplicates from list of modules - modules = sorted(list(set(modules))) - return (success, modules, metadata) - -def generate_suite_and_group_caps(suites, metadata_request, metadata_define, arguments, caps_dir, debug): - """Generate for the suite and for all groups parsed.""" - logging.info("Generating suite and group caps ...") - suite_and_group_caps = [] - # Change to caps directory - os.chdir(caps_dir) - for suite in suites: - logging.debug("Generating suite and group caps for suite {0}...".format(suite.name)) - # Write caps for suite and groups in suite - suite.write(metadata_request, metadata_define, arguments, debug) - suite_and_group_caps += suite.caps - os.chdir(BASEDIR) - if suite_and_group_caps: - success = True - else: - success = False - return (success, suite_and_group_caps) - -def generate_static_api(suites, static_api_dir, namespace): - """Generate static API for given suite(s)""" - success = True - # Change to caps directory, create if necessary - if not os.path.isdir(static_api_dir): - os.makedirs(static_api_dir) - os.chdir(static_api_dir) - api = API(suites=suites, directory=static_api_dir) - if namespace: - base = os.path.splitext(os.path.basename(api.filename))[0] - logging.info('Static API file name is ''{}'''.format(api.filename)) - api.filename = base+'_'+namespace+'.F90' - api.module = base+'_'+namespace - logging.info('Static API file name is changed to ''{}'''.format(api.filename)) - logging.info('Generating static API {0} in {1} ...'.format(api.filename, static_api_dir)) - api.write() - os.chdir(BASEDIR) - return (success, api) - -def generate_typedefs_makefile(metadata_define, typedefs_makefile, typedefs_cmakefile, typedefs_sourcefile): - """Generate list of Fortran modules containing CCPP type/kind definitions, - and create makefile/cmakefile snippets for host model build system""" - logging.info('Generating list of Fortran modules containing CCPP type definitions ...') - success = True - # - typedefs = [] - # (1) Search for type definitions in the metadata, defined by: - # (a) the type not being a standard type, and - # (b) the type not being the CCPP framework internal type - # (c) the standard_name being identical to the type name - # (2) Search for kind definitions in the metadata, defined by: - # (a) the standard_name starting with "kind_" - # (b) the type being integer and the units being none - for key in metadata_define.keys(): - # derived data types - if not metadata_define[key][0].type in STANDARD_VARIABLE_TYPES and \ - not metadata_define[key][0].type == CCPP_TYPE and \ - metadata_define[key][0].type == metadata_define[key][0].standard_name: - container = decode_container_as_dict(metadata_define[key][0].container) - if not 'MODULE' in container.keys(): - logging.error("Invalid type definition for type {}: {}".format(metadata_define[key][0].type, metadata_define[key][0].print_debug())) - success = False - continue - # Fortran modules are lowercase and have the ending ".mod" - typedef_fortran_module = "{}.mod".format(container['MODULE']).lower() - if not typedef_fortran_module in typedefs: - typedefs.append(typedef_fortran_module) - # kind definitions - elif metadata_define[key][0].standard_name.startswith("kind_") and \ - metadata_define[key][0].type == STANDARD_INTEGER_TYPE and \ - metadata_define[key][0].units == 'none': - container = decode_container_as_dict(metadata_define[key][0].container) - if not 'MODULE' in container.keys(): - logging.error("Invalid kind definition for kind {}: {}".format(metadata_define[key][0].type, metadata_define[key][0].print_debug())) - success = False - continue - # Fortran modules are lowercase and have the ending ".mod" - typedef_fortran_module = "{}.mod".format(container['MODULE']).lower() - if not typedef_fortran_module in typedefs: - typedefs.append(typedef_fortran_module) - - logging.info('Generating typedefs makefile/cmakefile snippet ...') - # Write the Fortran modules without path - the build system knows where they are - makefile = TypedefsMakefile() - makefile.filename = typedefs_makefile + '.tmp' - cmakefile = TypedefsCMakefile() - cmakefile.filename = typedefs_cmakefile + '.tmp' - sourcefile = TypedefsSourcefile() - sourcefile.filename = typedefs_sourcefile + '.tmp' - # Sort typedefs so that the order remains the same (for cmake to avoid) recompiling - typedefs.sort() - # Generate list of type definitions - makefile.write(typedefs) - cmakefile.write(typedefs) - sourcefile.write(typedefs) - if os.path.isfile(typedefs_makefile) and \ - filecmp.cmp(typedefs_makefile, makefile.filename): - os.remove(makefile.filename) - os.remove(cmakefile.filename) - os.remove(sourcefile.filename) - else: - if os.path.isfile(typedefs_makefile): - os.remove(typedefs_makefile) - if os.path.isfile(typedefs_cmakefile): - os.remove(typedefs_cmakefile) - if os.path.isfile(typedefs_sourcefile): - os.remove(typedefs_sourcefile) - os.rename(makefile.filename, typedefs_makefile) - os.rename(cmakefile.filename, typedefs_cmakefile) - os.rename(sourcefile.filename, typedefs_sourcefile) - # - logging.info('Added {0} typedefs to {1}, {2}, {3}'.format( - len(typedefs), typedefs_makefile, typedefs_cmakefile, typedefs_sourcefile)) - return success - -def generate_schemes_makefile(schemes, schemes_makefile, schemes_cmakefile, schemes_sourcefile): - """Generate makefile/cmakefile snippets for all schemes.""" - logging.info('Generating schemes makefile/cmakefile snippet ...') - success = True - makefile = SchemesMakefile() - makefile.filename = schemes_makefile + '.tmp' - cmakefile = SchemesCMakefile() - cmakefile.filename = schemes_cmakefile + '.tmp' - sourcefile = SchemesSourcefile() - sourcefile.filename = schemes_sourcefile + '.tmp' - # Sort schemes so that the order remains the same (for cmake to avoid) recompiling - schemes.sort() - # Generate list of schemes with absolute path - schemes_with_abspath = [ os.path.abspath(scheme) for scheme in schemes ] - makefile.write(schemes_with_abspath) - cmakefile.write(schemes_with_abspath) - sourcefile.write(schemes_with_abspath) - if os.path.isfile(schemes_makefile) and \ - filecmp.cmp(schemes_makefile, makefile.filename): - os.remove(makefile.filename) - os.remove(cmakefile.filename) - os.remove(sourcefile.filename) - else: - if os.path.isfile(schemes_makefile): - os.remove(schemes_makefile) - if os.path.isfile(schemes_cmakefile): - os.remove(schemes_cmakefile) - if os.path.isfile(schemes_sourcefile): - os.remove(schemes_sourcefile) - os.rename(makefile.filename, schemes_makefile) - os.rename(cmakefile.filename, schemes_cmakefile) - os.rename(sourcefile.filename, schemes_sourcefile) - # - logging.info('Added {0} schemes to {1}, {2}, {3}'.format( - len(schemes_with_abspath), schemes_makefile, schemes_cmakefile, schemes_sourcefile)) - return success - -def generate_caps_makefile(caps, caps_makefile, caps_cmakefile, caps_sourcefile, caps_dir): - """Generate makefile/cmakefile snippets for all caps.""" - logging.info('Generating caps makefile/cmakefile snippet ...') - success = True - makefile = CapsMakefile() - makefile.filename = caps_makefile + '.tmp' - cmakefile = CapsCMakefile() - cmakefile.filename = caps_cmakefile + '.tmp' - sourcefile = CapsSourcefile() - sourcefile.filename = caps_sourcefile + '.tmp' - # Sort caps so that the order remains the same (for cmake to avoid) recompiling - caps.sort() - # Generate list of caps with absolute path - caps_with_abspath = [ os.path.abspath(os.path.join(caps_dir, cap)) for cap in caps ] - makefile.write(caps_with_abspath) - cmakefile.write(caps_with_abspath) - sourcefile.write(caps_with_abspath) - if os.path.isfile(caps_makefile) and \ - filecmp.cmp(caps_makefile, makefile.filename): - os.remove(makefile.filename) - os.remove(cmakefile.filename) - os.remove(sourcefile.filename) - else: - if os.path.isfile(caps_makefile): - os.remove(caps_makefile) - if os.path.isfile(caps_cmakefile): - os.remove(caps_cmakefile) - if os.path.isfile(caps_sourcefile): - os.remove(caps_sourcefile) - os.rename(makefile.filename, caps_makefile) - os.rename(cmakefile.filename, caps_cmakefile) - os.rename(sourcefile.filename, caps_sourcefile) - # - logging.info('Added {0} auto-generated caps to {1} and {2}, {3}'.format( - len(caps_with_abspath), caps_makefile, caps_cmakefile, caps_sourcefile)) - return success - -def main(): - """Main routine that handles the CCPP prebuild for different host models.""" - # Parse command line arguments - (success, configfile, clean, verbose, debug, sdfs, builddir, namespace) = parse_arguments() - if not success: - raise Exception('Call to parse_arguments failed.') - - success = setup_logging(verbose) - if not success: - raise Exception('Call to setup_logging failed.') - - (success, config) = import_config(configfile, builddir) - if not success: - raise Exception('Call to import_config failed.') - - # Perform clean if requested, then exit - if clean: - success = clean_files(config, namespace) - logging.info('CCPP prebuild clean completed successfully, exiting.') - sys.exit(0) - - # Convert TYPEDEFS_NEW_METATA config to lowercase - config['typedefs_new_metadata'] = lowercase_keys_and_values(config['typedefs_new_metadata']) - - # If no suite definition files were given, get all of them - if not sdfs: - (success, sdfs) = get_all_suites(config['suites_dir']) - if not success: - raise Exception('Call to get_all_sdfs failed.') - - # Parse suite definition files for prebuild - (success, suites) = parse_suites(config['suites_dir'], sdfs) - if not success: - raise Exception('Parsing suite definition files failed.') - - # Variables defined by the host model - (success, metadata_define, dependencies_define) = gather_variable_definitions(config['variable_definition_files'], config['typedefs_new_metadata']) - if not success: - raise Exception('Call to gather_variable_definitions failed.') - - # Create an HTML table with all variables provided by the model - success = metadata_to_html(metadata_define, config['host_model'], config['html_vartable_file']) - if not success: - raise Exception('Call to metadata_to_html failed.') - - # Variables requested by the CCPP physics schemes - (success, metadata_request, arguments_request, dependencies_request, schemes_in_files) = collect_physics_subroutines(config['scheme_files']) - if not success: - raise Exception('Call to collect_physics_subroutines failed.') - - # Check that the schemes requested in the suites exist - success = check_schemes_in_suites(arguments_request, suites) - if not success: - raise Exception('Call to check_schemes_in_suites failed.') - - # Filter metadata/arguments - remove whatever is not included in suite definition files - (success, metadata_request, arguments_request, dependencies_request, schemes_in_files) = filter_metadata( - metadata_request, arguments_request, dependencies_request, schemes_in_files, suites) - if not success: - raise Exception('Call to filter_metadata failed.') - - # Add variables that are required to construct CCPP suites to the list of requested variables - (success, metadata_request) = add_ccpp_suite_variables(metadata_request) - if not success: - raise Exception('Call to add_ccpp_suite_variables failed.') - - (success, schemes_and_dependencies_to_compile) = generate_list_of_schemes_and_dependencies_to_compile( - schemes_in_files, dependencies_request, dependencies_define) - if not success: - raise Exception('Call to generate_list_of_schemes_and_dependencies_to_compile failed.') - - # Create a LaTeX table with all variables requested by the pool of physics and/or provided by the host model - success = metadata_to_latex(metadata_define, metadata_request, config['host_model'], config['latex_vartable_file']) - if not success: - raise Exception('Call to metadata_to_latex failed.') - - # Check requested against defined arguments to generate metadata (list/dict of variables for CCPP) - (success, modules, metadata) = compare_metadata(metadata_define, metadata_request) - if not success: - raise Exception('Call to compare_metadata failed.') - - # Add Fortran module files of typedefs to makefile/cmakefile/shell script - success = generate_typedefs_makefile(metadata_define, config['typedefs_makefile'], - config['typedefs_cmakefile'], config['typedefs_sourcefile']) - if not success: - raise Exception('Call to generate_typedefs_makefile failed.') - - # Add filenames of schemes and variable definition files (types) to makefile/cmakefile/shell script - success = generate_schemes_makefile(schemes_and_dependencies_to_compile + config['variable_definition_files'], - config['schemes_makefile'], config['schemes_cmakefile'], - config['schemes_sourcefile']) - if not success: - raise Exception('Call to generate_schemes_makefile failed.') - - # Static build: generate caps for entire suite and groups in the specified suite; generate API - (success, suite_and_group_caps) = generate_suite_and_group_caps(suites, metadata_request, metadata_define, - arguments_request, config['caps_dir'], debug) - if not success: - raise Exception('Call to generate_suite_and_group_caps failed.') - - (success, api) = generate_static_api(suites, config['static_api_dir'], namespace) - if not success: - raise Exception('Call to generate_static_api failed.') - - success = api.write_includefile(config['static_api_sourcefile'], type='shell') - if not success: - raise Exception("Writing API sourcefile {sourcefile} failed".format(sourcefile=config['static_api_sourcefile'])) - - success = api.write_includefile(config['static_api_cmakefile'], type='cmake') - if not success: - raise Exception("Writing API cmakefile {cmakefile} failed".format(cmakefile=config['static_api_cmakefile'])) - - # Add filenames of caps to makefile/cmakefile/shell script - all_caps = suite_and_group_caps - - success = generate_caps_makefile(all_caps, config['caps_makefile'], config['caps_cmakefile'], - config['caps_sourcefile'], config['caps_dir']) - if not success: - raise Exception('Call to generate_caps_makefile failed.') - - logging.info('CCPP prebuild step completed successfully.') - -if __name__ == '__main__': - main() diff --git a/scripts/ccpp_state_machine.py b/scripts/ccpp_state_machine.py deleted file mode 100644 index 832e0073..00000000 --- a/scripts/ccpp_state_machine.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Definition of the state machine used by the CCPP""" - -# CCPP framework imports -from state_machine import StateMachine - -_REG_ST = r"(?:register)" -_INIT_ST = r"(?:init(?:ial(?:ize)?)?)" -_FINAL_ST = r"(?:final(?:ize)?)" -_RUN_ST = r"(?:run)" -_TS_INIT_ST = r"(?:timestep_init(?:ial(?:ize)?)?)" -_TS_FINAL_ST = r"(?:timestep_final(?:ize)?)" - -# Allowed CCPP transitions -# pylint: disable=bad-whitespace -RUN_PHASE_NAME = 'run' -CCPP_STATE_MACH = StateMachine((('register', 'uninitialized', - 'uninitialized', _REG_ST), - ('initialize', 'uninitialized', - 'initialized', _INIT_ST), - ('timestep_initial', 'initialized', - 'in_time_step', _TS_INIT_ST), - (RUN_PHASE_NAME, 'in_time_step', - 'in_time_step', _RUN_ST), - ('timestep_final', 'in_time_step', - 'initialized', _TS_FINAL_ST), - ('finalize', 'initialized', - 'uninitialized', _FINAL_ST))) -# pylint: enable=bad-whitespace diff --git a/scripts/ccpp_suite.py b/scripts/ccpp_suite.py deleted file mode 100644 index d1a6e968..00000000 --- a/scripts/ccpp_suite.py +++ /dev/null @@ -1,1270 +0,0 @@ -#!/usr/bin/env python3 -# - -"""Classes and methods to create a Fortran suite-implementation file -to implement calls to a set of suites for a given host model.""" - -# Python library imports -import os.path -import logging -import xml.etree.ElementTree as ET -# CCPP framework imports -from ccpp_state_machine import CCPP_STATE_MACH, RUN_PHASE_NAME -from code_block import CodeBlock -from constituents import ConstituentVarDict -from ddt_library import DDTLibrary -from file_utils import KINDS_MODULE -from fortran_tools import FortranWriter -from framework_env import CCPPFrameworkEnv -from metavar import Var, VarDictionary, ccpp_standard_var -from parse_tools import ParseContext, ParseSource -from parse_tools import ParseInternalError, CCPPError -from parse_tools import read_xml_file, validate_xml_file, write_xml_file -from parse_tools import find_schema_version, expand_nested_suites -from parse_tools import init_log, set_log_to_null -from suite_objects import CallList, Group, Scheme -from metavar import CCPP_LOOP_VAR_STDNAMES -from var_props import is_horizontal_dimension - -# pylint: disable=too-many-lines - -############################################################################### -# Module (global) variables -############################################################################### - -# Source for internally generated variables. -API_SOURCE_NAME = "CCPP_API" -# Use the constituent source type for consistency -_API_SUITE_VAR_NAME = ConstituentVarDict.constitutent_source_type() -_API_SCHEME_VAR_NAME = "scheme" -_API_CONTEXT = ParseContext(filename="ccpp_suite.py") -_API_SOURCE = ParseSource(API_SOURCE_NAME, _API_SCHEME_VAR_NAME, _API_CONTEXT) -_API_LOGGING = init_log('ccpp_suite') -set_log_to_null(_API_LOGGING) -_API_DUMMY_RUN_ENV = CCPPFrameworkEnv(_API_LOGGING, - ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - -# Required variables for inclusion in auto-generated schemes -CCPP_REQUIRED_VARS = [ccpp_standard_var('ccpp_error_code', - _API_SCHEME_VAR_NAME, - _API_DUMMY_RUN_ENV, - context=_API_CONTEXT), - ccpp_standard_var('ccpp_error_message', - _API_SCHEME_VAR_NAME, - _API_DUMMY_RUN_ENV, - context=_API_CONTEXT)] - -############################################################################### - -class Suite(VarDictionary): - """Class to hold, process, and output a CAP for an entire CCPP suite. - The Suite includes initialization and finalization Group objects as - well as a Group for every suite part.""" - - __state_machine_initial_state = 'uninitialized' - __state_machine_var_name = 'ccpp_suite_state' - - __state_machine_init = ''' -character(len=16) :: {css_var_name} = '{state}' -''' - - # Note that these group names need to match CCPP_STATE_MACH - __register_group_name = 'register' - - __initial_group_name = 'initialize' - - __final_group_name = 'finalize' - - __timestep_initial_group_name = 'timestep_initial' - - __timestep_final_group_name = 'timestep_final' - - __scheme_template = '{}' - - def __init__(self, filename, suite_xml, api, run_env): - """Initialize this Suite object from the SDF, . - serves as the Suite's parent.""" - self.__run_env = run_env - self.__name = None - self.__sdf_name = filename - self.__groups = list() - self.__suite_init_group = None - self.__suite_final_group = None - self.__timestep_init_group = None - self.__timestep_final_group = None - self.__context = None - self.__host_arg_list_full = None - self.__host_arg_list_noloop = None - self.__module = None - self.__ddt_library = None - # Full phases/groups are special groups where the entire state is passed - self.__full_groups = {} - self._full_phases = {} - self.__gvar_stdnames = {} # Standard names of group-created vars - # Initialize our dictionary - # Create a 'parent' to hold the constituent variables - # The parent for the constituent dictionary is the API. - temp_name = os.path.splitext(os.path.basename(filename))[0] - const_dict = ConstituentVarDict(temp_name+'_constituents', - api, run_env) - super().__init__(self.sdf_name, run_env, parent_dict=const_dict) - if not os.path.exists(self.__sdf_name): - emsg = "Suite definition file {0} not found." - raise CCPPError(emsg.format(self.__sdf_name)) - # end if - # Parse the SDF - self.parse(suite_xml, run_env) - - @property - def name(self): - """Get the name of the suite.""" - return self.__name - - @property - def sdf_name(self): - """Get the name of the suite definition file.""" - return self.__sdf_name - - @classmethod - def check_suite_state(cls, stage): - """Return a list of CCPP state check statements for """ - check_stmts = list() - if stage in CCPP_STATE_MACH.transitions(): - # We need to make sure we are an allowed previous state - prev_state = CCPP_STATE_MACH.initial_state(stage) - css = "trim({})".format(Suite.__state_machine_var_name) - prev_str = "({} /= '{}')".format(css, prev_state) - check_stmts.append(("if {} then".format(prev_str), 1)) - check_stmts.append(("{errcode} = 1", 2)) - errmsg_str = "write({errmsg}, '(3a)') " - errmsg_str += "\"Invalid initial CCPP state, '\", " + css + ', ' - errmsg_str += "\"' in {funcname}\"" - check_stmts.append((errmsg_str, 2)) - check_stmts.append(("return", 2)) - check_stmts.append(("end if", 1)) - else: - raise ParseInternalError("Unknown stage, '{}'".format(stage)) - # end if - return CodeBlock(check_stmts) - - @classmethod - def set_suite_state(cls, phase): - """Return the code string to set the current suite state to . - If the initial and final states of are identical, return blank. - """ - initial = CCPP_STATE_MACH.initial_state(phase) - final = CCPP_STATE_MACH.final_state(phase) - if initial == final: - stmt = '! Suite state does not change' - else: - stmt = "ccpp_suite_state = '{}'".format(final) - # end if - return CodeBlock([(stmt, 1)]) - - def new_group(self, group_string, transition, run_env): - """Create a new Group object from the a XML description""" - if isinstance(group_string, str): - gxml = ET.fromstring(group_string) - else: - gxml = group_string - # end if - group = Group(gxml, transition, self, self.__context, run_env) - for svar in CCPP_REQUIRED_VARS: - group.add_call_list_variable(svar) - # end for - if transition != RUN_PHASE_NAME: - self.__full_groups[group.name] = group - self._full_phases[group.phase()] = group - # end if - return group - - def new_group_from_name(self, group_name, run_env): - '''Create an XML string for Group, , and use it to - create the corresponding group. - Note: must be the a transition string''' - group_xml = ''.format(group_name) - return self.new_group(group_xml, group_name, run_env) - - def parse(self, suite_xml, run_env): - """Parse the suite definition file.""" - success = True - # We do not have line number information for the XML file - self.__context = ParseContext(filename=self.__sdf_name) - self.__name = suite_xml.get('name') - self.__module = 'ccpp_{}_cap'.format(self.name) - gname = Suite.__register_group_name - self.__suite_reg_group = self.new_group_from_name(gname, run_env) - gname = Suite.__initial_group_name - self.__suite_init_group = self.new_group_from_name(gname, run_env) - gname = Suite.__final_group_name - self.__suite_final_group = self.new_group_from_name(gname, run_env) - gname = Suite.__timestep_initial_group_name - self.__timestep_init_group = self.new_group_from_name(gname, run_env) - gname = Suite.__timestep_final_group_name - self.__timestep_final_group = self.new_group_from_name(gname, run_env) - # Set up some groupings for later efficiency - self._beg_groups = [self.__suite_reg_group.name, - self.__suite_init_group.name, - self.__timestep_init_group.name] - self._end_groups = [self.__suite_final_group.name, - self.__timestep_final_group.name] - # Build hierarchical structure as in SDF - self.__groups.append(self.__suite_reg_group) - self.__groups.append(self.__suite_init_group) - self.__groups.append(self.__timestep_init_group) - for suite_item in suite_xml: - item_type = suite_item.tag.lower() - # Suite item is a group or a suite-wide init or final method - if item_type == 'group': - # Parse a group - self.__groups.append(self.new_group(suite_item, RUN_PHASE_NAME, - run_env)) - else: - match_trans = CCPP_STATE_MACH.function_match(item_type) - if match_trans is None: - emsg = "Unknown CCPP suite component tag type, '{}'" - raise CCPPError(emsg.format(item_type)) - # end if - if match_trans in self._full_phases: - # Parse a suite-wide initialization scheme - scheme = Scheme(suite_item, self.__context, - self, run_env) - self._full_phases[match_trans].add_item(scheme) - else: - emsg = "Unhandled CCPP suite component tag type, '{}'" - raise ParseInternalError(emsg.format(match_trans)) - # end if - # end for - self.__groups.append(self.__timestep_final_group) - self.__groups.append(self.__suite_final_group) - return success - - def suite_dicts(self): - """Return a list of this Suite's dictionaries. - A Suite's dictionaries are itself plus its constituent dictionary""" - return [self, self.parent] - - @property - def module(self): - """Get the list of the module generated for this suite.""" - return self.__module - - @property - def groups(self): - """Get the list of groups in this suite.""" - return self.__groups - - def find_variable(self, standard_name=None, source_var=None, - any_scope=True, clone=None, - search_call_list=False, loop_subst=False): - """Attempt to return the variable matching . - if is None, the standard name from is used. - It is an error to pass both and if - the standard name of is not the same as . - If is True, search parent scopes if not in current scope. - If the variable is not found this Suite's groups are searched for - a matching output variable. If found that variable is promoted to be a - Suite module variable and that variable is returned. - If the variable is not found and is not None, add a clone of - to this dictionary. - If the variable is not found and is None, return None. - """ - # First, see if the variable is already in our path - srch_clist = search_call_list - var = super().find_variable(standard_name=standard_name, - source_var=source_var, - any_scope=any_scope, - clone=None, - search_call_list=srch_clist, - loop_subst=loop_subst) - if var is None: - # No dice? Check for a group variable which can be promoted - # Don't promote loop standard names - if (standard_name in self.__gvar_stdnames and standard_name - not in CCPP_LOOP_VAR_STDNAMES): - group = self.__gvar_stdnames[standard_name] - var = group.find_variable(standard_name=standard_name, - source_var=source_var, - any_scope=False, - search_call_list=srch_clist, - loop_subst=loop_subst) - - if var is not None: - # Promote variable to suite level - # Remove this entry to avoid looping back here - del self.__gvar_stdnames[standard_name] - # Let everyone know this is now a Suite variable - var.source = ParseSource(API_SOURCE_NAME, - _API_SUITE_VAR_NAME, - var.context) - self.add_variable(var, self.__run_env) - # Remove the variable from the group - group.remove_variable(standard_name) - # Make sure the variable's dimensions are available - # at the init stage (for allocation) - for group in self.groups: - # only add dimension variables to init phase calling list - # if they're not module-level "suite" variables - if group.name == self.__suite_init_group.name: - dims = var.get_dimensions() - # replace horizontal loop dimension if necessary - for idx, dim in enumerate(dims): - if is_horizontal_dimension(dim): - if 'horizontal_loop' in dim: - dims[idx] = 'ccpp_constant_one:horizontal_dimension' - # end if - # end if - # end for - subst_dict = {'dimensions': dims} - prop_dict = var.copy_prop_dict(subst_dict=subst_dict) - temp_var = Var(prop_dict, - ParseSource(var.get_prop_value('scheme'), - var.get_prop_value('local_name'), var.context), - self.__run_env) - # Add dimensions if they're not already there - group.add_variable_dimensions(temp_var, [], - _API_SUITE_VAR_NAME, - adjust_intent=True, - to_dict=group.call_list) - # end if - # end for - else: - emsg = ("Group, {}, claimed it had created {} " - "but variable was not found") - raise CCPPError(emsg.format(group.name, standard_name)) - # end if - # end if - # end if - if (var is None) and (clone is not None): - # Guess it is time to clone a different variable - var = super().find_variable(standard_name=standard_name, - source_var=source_var, - any_scope=any_scope, clone=clone) - # end if - return var - - def analyze(self, host_model, scheme_library, ddt_library, run_env): - """Collect all information needed to write a suite file - >>> CCPP_STATE_MACH.transition_match('init') - 'initialize' - >>> CCPP_STATE_MACH.transition_match('init', transition='finalize') - - >>> CCPP_STATE_MACH.transition_match('INIT') - 'initialize' - >>> CCPP_STATE_MACH.transition_match('initial') - 'initialize' - >>> CCPP_STATE_MACH.transition_match('timestep_initial') - 'timestep_initial' - >>> CCPP_STATE_MACH.transition_match('timestep_initialize') - 'timestep_initial' - >>> CCPP_STATE_MACH.transition_match('timestep_init') - 'timestep_initial' - >>> CCPP_STATE_MACH.transition_match('initialize') - 'initialize' - >>> CCPP_STATE_MACH.transition_match('initialize')[0:4] - 'init' - >>> CCPP_STATE_MACH.transition_match('initize') - - >>> CCPP_STATE_MACH.transition_match('run') - 'run' - >>> CCPP_STATE_MACH.transition_match('finalize') - 'finalize' - >>> CCPP_STATE_MACH.transition_match('finalize')[0:5] - 'final' - >>> CCPP_STATE_MACH.transition_match('final') - 'finalize' - >>> CCPP_STATE_MACH.transition_match('finalize_bar') - - >>> CCPP_STATE_MACH.function_match('foo_init') - ('foo', 'init', 'initialize') - >>> CCPP_STATE_MACH.function_match('foo_init', transition='finalize') - (None, None, None) - >>> CCPP_STATE_MACH.function_match('FOO_INIT') - ('FOO', 'INIT', 'initialize') - >>> CCPP_STATE_MACH.function_match('foo_initial') - ('foo', 'initial', 'initialize') - >>> CCPP_STATE_MACH.function_match('foo_initialize') - ('foo', 'initialize', 'initialize') - >>> CCPP_STATE_MACH.function_match('foo_initialize')[1][0:4] - 'init' - >>> CCPP_STATE_MACH.function_match('foo_initize') - (None, None, None) - >>> CCPP_STATE_MACH.function_match('foo_timestep_initial') - ('foo', 'timestep_initial', 'timestep_initial') - >>> CCPP_STATE_MACH.function_match('foo_timestep_init') - ('foo', 'timestep_init', 'timestep_initial') - >>> CCPP_STATE_MACH.function_match('foo_timestep_initialize') - ('foo', 'timestep_initialize', 'timestep_initial') - >>> CCPP_STATE_MACH.function_match('foo_run') - ('foo', 'run', 'run') - >>> CCPP_STATE_MACH.function_match('foo_finalize') - ('foo', 'finalize', 'finalize') - >>> CCPP_STATE_MACH.function_match('foo_finalize')[1][0:5] - 'final' - >>> CCPP_STATE_MACH.function_match('foo_final') - ('foo', 'final', 'finalize') - >>> CCPP_STATE_MACH.function_match('foo_finalize_bar') - (None, None, None) - >>> CCPP_STATE_MACH.function_match('foo_timestep_final') - ('foo', 'timestep_final', 'timestep_final') - >>> CCPP_STATE_MACH.function_match('foo_timestep_finalize') - ('foo', 'timestep_finalize', 'timestep_final') - """ - self.__ddt_library = ddt_library - # Collect all relevant schemes - # For all groups, find associated init and final methods - scheme_list = list() - for group in self.groups: - for scheme in group.schemes(): - scheme_list.append(scheme.name) - # end for - # end for - no_scheme_entries = {} # Skip schemes that are not in this suite - for module in scheme_list: - if scheme_library[module]: - scheme_entries = scheme_library[module] - else: - scheme_entries = no_scheme_entries - # end if - for phase in self._full_phases: - if phase in scheme_entries: - header = scheme_entries[phase] - # Add this scheme's init or final routine - pgroup = self._full_phases[phase] - if not pgroup.has_item(header.title): - sstr = Suite.__scheme_template.format(module) - sxml = ET.fromstring(sstr) - scheme = Scheme(sxml, self.__context, pgroup, run_env) - pgroup.add_part(scheme) - # end if (no else, scheme is already in group) - # end if (no else, phase not in scheme set) - # end for - # end for - # Grab the host model argument list - self.__host_arg_list_full = host_model.argument_list() - self.__host_arg_list_noloop = host_model.argument_list(loop_vars=False) - # First pass, create init, run, and finalize sequences - for item in self.groups: - if item.name in self.__full_groups: - phase = self.__full_groups[item.name].phase() - else: - phase = RUN_PHASE_NAME - # end if - lmsg = "Group {}, schemes = {}" - if run_env.verbose: - run_env.logger.debug(lmsg.format(item.name, - [x.name - for x in item.schemes()])) - item.analyze(phase, self, scheme_library, ddt_library, - self.check_suite_state(phase), - self.set_suite_state(phase)) - # Look for group variables that need to be promoted to the suite - # We need to promote any variable used later to the suite, however, - # we do not yet know if it will be used. - # Add new group-created variables - gvars = item.variable_list() - for gvar in gvars: - stdname = gvar.get_prop_value('standard_name') - if not stdname in self.__gvar_stdnames: - self.__gvar_stdnames[stdname] = item - # end if - # end for - # end for - - def is_run_group(self, group): - """Method to separate out run-loop groups from special initial - and final groups - """ - return ((group.name not in self._beg_groups) and - (group.name not in self._end_groups)) - - def max_part_len(self): - """What is the longest suite subroutine name?""" - maxlen = 0 - for spart in self.groups: - if self.is_run_group(spart): - maxlen = max(maxlen, len(spart.name)) - # end if - # end for - return maxlen - - def part_list(self): - """Return list of run phase parts (groups)""" - parts = list() - for spart in self.groups: - if self.is_run_group(spart): - parts.append(spart.name[len(self.name)+1:]) - # end if - # end for - return parts - - def phase_group(self, phase): - """Return the (non-run) group specified by """ - if phase in self._full_phases: - return self._full_phases[phase] - # end if - raise ParseInternalError("Incorrect phase, '{}'".format(phase)) - - def constituent_dictionary(self): - """Return the constituent dictionary for this suite""" - return self.parent - - def write(self, output_dir, run_env): - """Create caps for all groups in the suite and for the entire suite - (calling the group caps one after another)""" - # Set name of module and filename of cap - filename = '{module_name}.F90'.format(module_name=self.module) - if run_env.verbose: - run_env.logger.debug('Writing CCPP suite file, {}'.format(filename)) - # end if - # Retrieve the name of the constituent module for Group use statements - const_mod = self.parent.constituent_module_name() - # Init - output_file_name = os.path.join(output_dir, filename) - with FortranWriter(output_file_name, 'w', - "CCPP Suite Cap for {}".format(self.name), - self.module) as outfile: - # Write module 'use' statements here - outfile.write('use {}'.format(KINDS_MODULE), 1) - # Look for any DDT types - self.__ddt_library.write_ddt_use_statements(self.values(), - outfile, 1) - # Write out constituent module use statement(s) - const_dict = self.constituent_dictionary() - const_dict.write_suite_use(outfile, 1) - outfile.write_preamble() - outfile.write('! Suite interfaces', 1) - line = Suite.__state_machine_init - var_name = Suite.__state_machine_var_name - var_state = Suite.__state_machine_initial_state - outfile.write(line.format(css_var_name=var_name, - state=var_state), 1) - for group in self.__groups: - outfile.write('public :: {}'.format(group.name), 1) - # end for - # Declare constituent public interfaces - const_dict.declare_public_interfaces(outfile, 1) - # Declare constituent private suite interfaces and data - const_dict.declare_private_data(outfile, 1) - outfile.write('\n! Private suite variables', 1) - for svar in self.keys(): - self[svar].write_def(outfile, 1, self, allocatable=True) - # end for - outfile.end_module_header() - for group in self.__groups: - if group.name in self._beg_groups: - if group.name == self.__suite_reg_group.name: - group.write(outfile, self.__host_arg_list_noloop, - 1, const_mod, suite_vars=self) - else: - group.write(outfile, self.__host_arg_list_noloop, - 1, const_mod, suite_vars=self, allocate=True) - # end if - elif group.name in self._end_groups: - group.write(outfile, self.__host_arg_list_noloop, - 1, const_mod, suite_vars=self, deallocate=True) - else: - group.write(outfile, self.__host_arg_list_full, 1, - const_mod) - # end if - # end for - err_vars = self.find_error_variables(any_scope=True, - clone_as_out=True) - # Write the constituent properties interface - const_dict.write_constituent_routines(outfile, 1, - self.name, err_vars) - # end with - return output_file_name - -############################################################################### - -class API(VarDictionary): - """Class representing the API for the CCPP framework. - The API class organizes the suites for which CAPS will be generated""" - - __suite_fname = 'ccpp_physics_suite_list' - __part_fname = 'ccpp_physics_suite_part_list' - __vars_fname = 'ccpp_physics_suite_variables' - __schemes_fname = 'ccpp_physics_suite_schemes' - - __file_desc = "API for {host_model} calls to CCPP suites" - - __preamble = ''' -{module_use} -''' - - __sub_name_template = 'ccpp_physics' - - __subhead = 'subroutine {subname}({api_call_list})' - - __subfoot = 'end subroutine {subname}\n' - - # Note, we cannot add these vars to our dictionary as we do not want - # them showing up in group dummy arg lists - __suite_name = Var({'local_name':'suite_name', - 'standard_name':'suite_name', - 'intent':'in', 'type':'character', - 'kind':'len=*', 'units':'', - 'dimensions':'()'}, _API_SOURCE, _API_DUMMY_RUN_ENV) - - __suite_part = Var({'local_name':'suite_part', - 'standard_name':'suite_part', - 'intent':'in', 'type':'character', - 'kind':'len=*', 'units':'', - 'dimensions':'()'}, _API_SOURCE, _API_DUMMY_RUN_ENV) - - def __init__(self, sdfs, host_model, scheme_headers, run_env): - """Initialize this API. - is the list of Suite Definition Files to be parsed for - data needed by the CCPP cap. - is a HostModel object to reference for host model - variables. - is the list of parsed physics scheme metadata files. - Every scheme referenced by an SDF in MUST be in this list, - however, unused schemes are allowed. - is the CCPPFrameworkEnv object for this framework run. - """ - self.__module = 'ccpp_physics_api' - self.__host = host_model - self.__suites = list() - super().__init__(self.module, run_env, parent_dict=self.host_model) - # Create a usable library out of scheme_headers - # Structure is dictionary of dictionaries - # Top-level dictionary is keyed by function name - # Secondary level is by phase - scheme_library = {} - # First, process DDT headers - all_ddts = [d for d in scheme_headers if d.header_type == 'ddt'] - ddt_titles = [d.title for d in all_ddts] - for ddt_title in self.host_model.ddt_lib: - if ddt_title not in ddt_titles: - all_ddts.append(self.host_model.ddt_lib[ddt_title]) - # end if - # end for - self.__ddt_lib = DDTLibrary('{}_api'.format(self.host_model.name), - run_env, ddts=all_ddts) - for header in [d for d in scheme_headers if d.header_type != 'ddt']: - if header.header_type != 'scheme': - if header.header_type == 'module': - errmsg = f"{header.title} is a module metadata header type." - errmsg+=" This is not an allowed CCPP scheme header type." - else: - errmsg = f"{header.title} is an unknown CCPP API metadata header type, {header.header_type}" - # end if - raise CCPPError(errmsg) - # end if - func_id, _, match_trans = \ - CCPP_STATE_MACH.function_match(header.title) - if func_id not in scheme_library: - scheme_library[func_id] = {} - # end if - func_entry = scheme_library[func_id] - if match_trans not in func_entry: - func_entry[match_trans] = header - else: - errmsg = "Duplicate scheme entry, {}" - raise CCPPError(errmsg.format(header.title)) - # end if - # end for - - # Turn the SDF files into Suites - for sdf in sdfs: - # Load the suite definition file to determine the schema version, - # validate the file, and expand nested suites if applicable - _, xml_root = read_xml_file(sdf, run_env.logger) - # We do not have line number information for the XML file - self.__context = ParseContext(filename=sdf) - # Validate the XML file - schema_version = find_schema_version(xml_root) - _ = validate_xml_file(sdf, 'suite', schema_version, run_env.logger) - - # Write the expanded sdf to the capgen output directory. - # This file isn't used by capgen (everything is in memory - # from here onwards), but it is useful for developers/users - # (although the output can also be found in the datatable). - (sdf_path, sdf_name) = os.path.split(sdf) - sdf_expanded = os.path.join(run_env.output_dir, - sdf_name.replace(".xml", "_expanded.xml")) - if schema_version[0] in [1, 2]: - # Preprocess the sdf to expand nested suites - if schema_version[0] == 2: - expand_nested_suites(xml_root, sdf_path, logger=run_env.logger) - # For both versions 1 and 2, write the SDF (expanded for - # version 2, original for version 1) to the current directory - write_xml_file(xml_root, sdf_expanded, run_env.logger) - # Validate the expanded SDF for version 2 - if schema_version[0] == 2: - _ = validate_xml_file(sdf, 'suite', schema_version, run_env.logger) - suite = Suite(sdf, xml_root, self, run_env) - suite.analyze(self.host_model, scheme_library, - self.__ddt_lib, run_env) - self.__suites.append(suite) - else: - errmsg = f"Suite XML schema not supported: " + \ - "root={xml_root.tag}, version={schema_version}" - raise CCPPError(errmsg) - # end if - # end for - - # We will need the correct names for errmsg and errcode - evar = self.host_model.find_variable(standard_name='ccpp_error_message') - if evar is not None: - self._errmsg_var = evar - else: - raise CCPPError('Required variable, ccpp_error_message, not found') - # end if - evar = self.host_model.find_variable(standard_name='ccpp_error_code') - if evar is not None: - self._errcode_var = evar - else: - raise CCPPError('Required variable, ccpp_error_code, not found') - # end if - # We need a call list for every phase - self.__call_lists = {} - for phase in CCPP_STATE_MACH.transitions(): - self.__call_lists[phase] = CallList('API_' + phase, run_env) - self.__call_lists[phase].add_variable(self.suite_name_var, run_env) - if phase == RUN_PHASE_NAME: - self.__call_lists[phase].add_variable(self.suite_part_var, - run_env) - # end if - for suite in self.__suites: - for group in suite.groups: - if group.phase() == phase: - self.__call_lists[phase].add_vars(group.call_list, - run_env, - gen_unique=True) - # end if - # end for - # end for - # end for - - @classmethod - def interface_name(cls, phase): - 'Return the name of an API interface function' - return "{}_{}".format(cls.__sub_name_template, phase) - - def call_list(self, phase): - "Return the appropriate API call list variables" - if phase in self.__call_lists: - return self.__call_lists[phase] - # end if - raise ParseInternalError("Illegal phase, '{}'".format(phase)) - - def write(self, output_dir, run_env): - """Write CCPP API module""" - if not self.suites: - raise CCPPError("No suite specified for generating API") - # end if - api_filenames = list() - # Write out the suite files - for suite in self.suites: - out_file_name = suite.write(output_dir, run_env) - api_filenames.append(out_file_name) - # end for - return api_filenames - - @classmethod - def declare_inspection_interfaces(cls, ofile): - """Declare the API interfaces for the suite inquiry functions""" - ofile.write(f"public :: {API.__suite_fname}", 1) - ofile.write(f"public :: {API.__part_fname}", 1) - ofile.write(f"public :: {API.__vars_fname}", 1) - ofile.write(f"public :: {API.__schemes_fname}", 1) - - def get_errinfo_names(self, base_only=False): - """Return a tuple of error output local names. - If base_only==True, return only the name string of the variable. - If base_only=False, return the local name as a full reference. - If the error variables are intrinsic variables, this makes no - difference, however, for a DDT variable, the full reference is - % while the local name is just .""" - if base_only: - errmsg_name = self._errmsg_var.get_prop_value('local_name') - errcode_name = self._errcode_var.get_prop_value('local_name') - else: - errmsg_name = self._errmsg_var.call_string(self) - errcode_name = self._errcode_var.call_string(self) - # end if - return (errmsg_name, errcode_name) - - @staticmethod - def write_var_set_loop(ofile, varlist_name, var_list, indent, - add_allocate=True, start_index=1, start_var=None): - """Write code to allocate (if is True) and set - to . Elements of are set - beginning at . - """ - if add_allocate: - ofile.write(f"allocate({varlist_name}({len(var_list)}))", indent) - # end if - for ind, var in enumerate(var_list): - if start_var: - ind_str = f"{start_var} + {ind + start_index}" - else: - ind_str = f"{ind + start_index}" - # end if - ofile.write(f"{varlist_name}({ind_str}) = '{var}'", indent) - # end for - - def write_suite_part_list_sub(self, ofile, errmsg_name, errcode_name): - """Write the suite-part list subroutine""" - inargs = f"suite_name, part_list, {errmsg_name}, {errcode_name}" - ofile.write(f"subroutine {API.__part_fname}({inargs})", 1) - oline = "character(len=*), intent(in) :: suite_name" - ofile.write(oline, 2) - oline = "character(len=*), allocatable, intent(out) :: part_list(:)" - ofile.write(oline, 2) - self._errmsg_var.write_def(ofile, 2, self, dummy=True, add_intent="out", - extra_space=11) - self._errcode_var.write_def(ofile, 2, self, dummy=True, add_intent="out", - extra_space=11) - else_str = '' - ename = self._errcode_var.get_prop_value('local_name') - ofile.write(f"{ename} = 0", 2) - ename = self._errmsg_var.get_prop_value('local_name') - ofile.write(f"{ename} = ''", 2) - for suite in self.suites: - oline = "{}if(trim(suite_name) == '{}') then" - ofile.write(oline.format(else_str, suite.name), 2) - API.write_var_set_loop(ofile, 'part_list', suite.part_list(), 3) - else_str = 'else ' - # end for - ofile.write("else", 2) - emsg = f"write({errmsg_name}, '(3a)')" - emsg += "'No suite named ', trim(suite_name), ' found'" - ofile.write(emsg, 3) - ofile.write(f"{errcode_name} = 1", 3) - ofile.write("end if", 2) - ofile.write(f"end subroutine {API.__part_fname}", 1) - - def write_req_vars_sub(self, ofile, errmsg_name, errcode_name): - """Write the required variables subroutine""" - oline = "suite_name, variable_list, {errmsg}, {errcode}" - oline += ", input_vars, output_vars, struct_elements" - inargs = oline.format(errmsg=errmsg_name, errcode=errcode_name) - ofile.write("\nsubroutine {}({})".format(API.__vars_fname, inargs), 1) - ofile.write("! Dummy arguments", 2) - oline = "character(len=*), intent(in) :: suite_name" - ofile.write(oline, 2) - oline = "character(len=*), allocatable, intent(out) :: variable_list(:)" - ofile.write(oline, 2) - self._errmsg_var.write_def(ofile, 2, self, dummy=True, - add_intent="out", extra_space=11) - self._errcode_var.write_def(ofile, 2, self, dummy=True, - add_intent="out", extra_space=11) - oline = "logical, optional, intent(in) :: input_vars" - ofile.write(oline, 2) - oline = "logical, optional, intent(in) :: output_vars" - ofile.write(oline, 2) - oline = "logical, optional, intent(in) :: struct_elements" - ofile.write(oline, 2) - ofile.write("! Local variables", 2) - ofile.write("logical {}:: input_vars_use".format(' '*34), 2) - ofile.write("logical {}:: output_vars_use".format(' '*34), 2) - ofile.write("logical {}:: struct_elements_use".format(' '*34), 2) - ofile.write("integer {}:: num_vars".format(' '*34), 2) - ofile.write("", 0) - ename = self._errcode_var.get_prop_value('local_name') - ofile.write("{} = 0".format(ename), 2) - ename = self._errmsg_var.get_prop_value('local_name') - ofile.write("{} = ''".format(ename), 2) - ofile.write("if (present(input_vars)) then", 2) - ofile.write("input_vars_use = input_vars", 3) - ofile.write("else", 2) - ofile.write("input_vars_use = .true.", 3) - ofile.write("end if", 2) - ofile.write("if (present(output_vars)) then", 2) - ofile.write("output_vars_use = output_vars", 3) - ofile.write("else", 2) - ofile.write("output_vars_use = .true.", 3) - ofile.write("end if", 2) - ofile.write("if (present(struct_elements)) then", 2) - ofile.write("struct_elements_use = struct_elements", 3) - ofile.write("else", 2) - ofile.write("struct_elements_use = .true.", 3) - ofile.write("end if", 2) - else_str = '' - for suite in self.suites: - parent = suite.parent - # Collect all the suite variables - oline = "{}if(trim(suite_name) == '{}') then" - input_vars = [set(), set(), set()] # leaves, arrays, leaf elements - inout_vars = [set(), set(), set()] # leaves, arrays, leaf elements - output_vars = [set(), set(), set()] # leaves, arrays, leaf elements - const_initialized_in_physics = {} - for part in suite.groups: - for var in part.call_list.variable_list(): - phase = part.phase() - stdname = var.get_prop_value("standard_name") - intent = var.get_prop_value("intent") - protected = var.get_prop_value("protected") - constituent = var.is_constituent() - if stdname not in const_initialized_in_physics: - const_initialized_in_physics[stdname] = False - # end if - if (parent is not None) and (not protected): - pvar = parent.find_variable(standard_name=stdname) - if pvar is not None: - protected = pvar.get_prop_value("protected") - # end if - # end if - elements = var.intrinsic_elements(check_dict=self.parent, - ddt_lib=self.__ddt_lib) - if (intent == 'in') and (not protected) and (not const_initialized_in_physics[stdname]): - if isinstance(elements, list): - input_vars[1].add(stdname) - input_vars[2].update(elements) - else: - input_vars[0].add(stdname) - # end if - elif intent == 'inout' and (not const_initialized_in_physics[stdname]): - if isinstance(elements, list): - inout_vars[1].add(stdname) - inout_vars[2].update(elements) - else: - inout_vars[0].add(stdname) - # end if - elif constituent and (intent == 'out' and phase != 'initialize' and not - const_initialized_in_physics[stdname]): - # constituents HAVE to be initialized in the init phase because the dycore needs to advect them - emsg = f"constituent variable '{stdname}' cannot be initialized in the '{phase}' phase" - raise CCPPError(emsg) - elif intent == 'out' and constituent and phase == 'initialize': - const_initialized_in_physics[stdname] = True - elif intent == 'out': - if isinstance(elements, list): - output_vars[1].add(stdname) - output_vars[2].update(elements) - else: - output_vars[0].add(stdname) - # end if - # end if - # end for - # end for - # Figure out how many total variables to return and allocate - # variable_list to that size - ofile.write(oline.format(else_str, suite.name), 2) - ofile.write("if (input_vars_use .and. output_vars_use) then", 3) - have_elems = input_vars[2] or inout_vars[2] or output_vars[2] - if have_elems: - ofile.write("if (struct_elements_use) then", 4) - numvars = len(input_vars[0] | input_vars[2] | inout_vars[0] | - inout_vars[2] | output_vars[0] | output_vars[2]) - ofile.write("num_vars = {}".format(numvars), 5) - ofile.write("else", 4) - # end if - numvars = len(input_vars[0] | input_vars[1] | inout_vars[0] | - inout_vars[1] | output_vars[0] | output_vars[1]) - ofile.write("num_vars = {}".format(numvars), 5 if have_elems else 4) - if have_elems: - ofile.write("end if", 4) - # end if - ofile.write("else if (input_vars_use) then", 3) - have_elems = input_vars[2] or inout_vars[2] - if have_elems: - ofile.write("if (struct_elements_use) then", 4) - numvars = len(input_vars[0] | input_vars[2] | - inout_vars[0] | inout_vars[2]) - ofile.write("num_vars = {}".format(numvars), 5) - ofile.write("else", 4) - # end if - numvars = len(input_vars[0] | input_vars[1] | - inout_vars[0] | inout_vars[1]) - ofile.write("num_vars = {}".format(numvars), 5 if have_elems else 4) - if have_elems: - ofile.write("end if", 4) - # end if - ofile.write("else if (output_vars_use) then", 3) - have_elems = inout_vars[2] or output_vars[2] - if have_elems: - ofile.write("if (struct_elements_use) then", 4) - numvars = len(inout_vars[0] | inout_vars[2] | - output_vars[0] | output_vars[2]) - ofile.write("num_vars = {}".format(numvars), 5) - ofile.write("else", 4) - # end if - numvars = len(inout_vars[0] | inout_vars[1] | - output_vars[0] | output_vars[1]) - ofile.write("num_vars = {}".format(numvars), 5 if have_elems else 4) - if have_elems: - ofile.write("end if", 4) - # end if - ofile.write("else", 3) - ofile.write("num_vars = 0", 4) - ofile.write("end if", 3) - ofile.write("allocate(variable_list(num_vars))", 3) - # Now, fill in the variable_list array - # Start with inout variables - elem_start = 1 - leaf_start = 1 - leaf_written_set = inout_vars[0].copy() - elem_written_set = inout_vars[0].copy() - leaf_list = sorted(inout_vars[0]) - if inout_vars[0] or inout_vars[1] or inout_vars[2]: - ofile.write("if (input_vars_use .or. output_vars_use) then", 3) - API.write_var_set_loop(ofile, 'variable_list', leaf_list, 4, - add_allocate=False, - start_index=leaf_start) - # end if - leaf_start += len(leaf_list) - elem_start += len(leaf_list) - # elements which have not been written out - elem_list = sorted(inout_vars[2] - elem_written_set) - elem_written_set = elem_written_set | inout_vars[2] - leaf_list = sorted(inout_vars[1] - leaf_written_set) - leaf_written_set = leaf_written_set | inout_vars[1] - if elem_list or leaf_list: - ofile.write("if (struct_elements_use) then", 4) - API.write_var_set_loop(ofile, 'variable_list', elem_list, 5, - add_allocate=False, - start_index=elem_start) - elem_start += len(elem_list) - ofile.write("num_vars = {}".format(elem_start - 1), 5) - ofile.write("else", 4) - API.write_var_set_loop(ofile, 'variable_list', leaf_list, 5, - add_allocate=False, - start_index=leaf_start) - leaf_start += len(leaf_list) - ofile.write("num_vars = {}".format(leaf_start - 1), 5) - ofile.write("end if", 4) - else: - ofile.write("num_vars = {}".format(len(leaf_written_set)), - 4 if leaf_written_set else 3) - # end if - if inout_vars[0] or inout_vars[1] or inout_vars[2]: - ofile.write("end if", 3) - # end if - # Write input variables - leaf_list = sorted(input_vars[0] - leaf_written_set) - # Are there any output variables which are also input variables - # (e.g., for a different part (group) of the suite)? - # We need to collect them now in case is selected - # but not . - leaf_cross_set = output_vars[0] & input_vars[0] - simp_cross_set = (output_vars[1] & input_vars[1]) - leaf_cross_set - elem_cross_set = (output_vars[2] & input_vars[2]) - leaf_cross_set - # Subtract the variables which have already been written out - leaf_cross_list = sorted(leaf_cross_set - leaf_written_set) - simp_cross_list = sorted(simp_cross_set - leaf_written_set) - elem_cross_list = sorted(elem_cross_set - elem_written_set) - # Next move back to processing the input variables - leaf_written_set = leaf_written_set | input_vars[0] - elem_list = sorted(input_vars[2] - elem_written_set) - elem_written_set = elem_written_set | input_vars[0] | input_vars[2] - have_inputs = elem_list or leaf_list - if have_inputs: - ofile.write("if (input_vars_use) then", 3) - # elements which have not been written out - # end if - API.write_var_set_loop(ofile, 'variable_list', leaf_list, 4, - add_allocate=False, start_var="num_vars", - start_index=1) - if leaf_list: - ofile.write("num_vars = num_vars + {}".format(len(leaf_list)), - 4) - # end if - leaf_start += len(leaf_list) - elem_start += len(leaf_list) - leaf_list = input_vars[1].difference(leaf_written_set) - leaf_written_set.union(input_vars[1]) - if elem_list or leaf_list: - ofile.write("if (struct_elements_use) then", 4) - API.write_var_set_loop(ofile, 'variable_list', elem_list, 5, - add_allocate=False, - start_index=elem_start) - elem_start += len(elem_list) - 1 - ofile.write("num_vars = {}".format(elem_start), 5) - ofile.write("else", 4) - API.write_var_set_loop(ofile, 'variable_list', leaf_list, 5, - add_allocate=False, - start_index=leaf_start) - leaf_start += len(leaf_list) - 1 - ofile.write("num_vars = {}".format(leaf_start), 5) - ofile.write("end if", 4) - # end if - if have_inputs: - ofile.write("end if", 3) - # end if - # Write output variables - leaf_list = sorted(output_vars[0].difference(leaf_written_set)) - leaf_written_set = leaf_written_set.union(output_vars[0]) - elem_written_set = elem_written_set.union(output_vars[0]) - elem_list = sorted(output_vars[2].difference(elem_written_set)) - elem_written_set = elem_written_set.union(output_vars[2]) - have_outputs = elem_list or leaf_list - if have_outputs: - ofile.write("if (output_vars_use) then", 3) - # end if - leaf_start = 1 - API.write_var_set_loop(ofile, 'variable_list', leaf_list, 4, - add_allocate=False, start_var="num_vars", - start_index=leaf_start) - leaf_start += len(leaf_list) - elem_start = leaf_start - leaf_list = output_vars[1].difference(leaf_written_set) - leaf_written_set.union(output_vars[1]) - if elem_list or leaf_list: - ofile.write("if (struct_elements_use) then", 4) - API.write_var_set_loop(ofile, 'variable_list', elem_list, 5, - add_allocate=False, start_var="num_vars", - start_index=elem_start) - elem_start += len(elem_list) - ofile.write("else", 4) - API.write_var_set_loop(ofile, 'variable_list', leaf_list, 5, - add_allocate=False, start_var="num_vars", - start_index=leaf_start) - leaf_start += len(leaf_list) - ofile.write("end if", 4) - # end if - if leaf_cross_list or elem_cross_list: - ofile.write("if (.not. input_vars_use) then", 4) - API.write_var_set_loop(ofile, 'variable_list', leaf_cross_list, - 5, add_allocate=False, - start_var="num_vars", - start_index=leaf_start) - leaf_start += len(leaf_cross_list) - elem_start += len(leaf_cross_list) - if elem_cross_list or simp_cross_list: - ofile.write("if (struct_elements_use) then", 5) - API.write_var_set_loop(ofile, 'variable_list', - elem_cross_list, 6, - add_allocate=False, - start_var="num_vars", - start_index=elem_start) - elem_start += len(elem_list) - ofile.write("else", 5) - API.write_var_set_loop(ofile, 'variable_list', - leaf_cross_list, 6, - add_allocate=False, - start_var="num_vars", - start_index=leaf_start) - leaf_start += len(leaf_list) - ofile.write("end if", 5) - # end if - ofile.write("end if", 4) - if have_outputs: - ofile.write("end if", 3) - # end if - else_str = 'else ' - # end for - ofile.write("else", 2) - emsg = "write({errmsg}, '(3a)')".format(errmsg=errmsg_name) - emsg += "'No suite named ', trim(suite_name), ' found'" - ofile.write(emsg, 3) - ofile.write("{errcode} = 1".format(errcode=errcode_name), 3) - ofile.write("end if", 2) - ofile.write("end subroutine {}".format(API.__vars_fname), 1) - - def write_suite_schemes_sub(self, ofile, errmsg_name, errcode_name): - """Write the suite schemes list subroutine""" - oline = "suite_name, scheme_list, {errmsg}, {errcode}" - inargs = oline.format(errmsg=errmsg_name, errcode=errcode_name) - ofile.write("\nsubroutine {}({})".format(API.__schemes_fname, - inargs), 1) - oline = "character(len=*), intent(in) :: suite_name" - ofile.write(oline, 2) - oline = "character(len=*), allocatable, intent(out) :: scheme_list(:)" - ofile.write(oline, 2) - self._errmsg_var.write_def(ofile, 2, self, dummy=True, - add_intent="out", extra_space=11) - self._errcode_var.write_def(ofile, 2, self, dummy=True, - add_intent="out", extra_space=11) - else_str = '' - ename = self._errcode_var.get_prop_value('local_name') - ofile.write("{} = 0".format(ename), 2) - ename = self._errmsg_var.get_prop_value('local_name') - ofile.write("{} = ''".format(ename), 2) - for suite in self.suites: - oline = "{}if(trim(suite_name) == '{}') then" - ofile.write(oline.format(else_str, suite.name), 2) - # Collect the list of schemes in this suite - schemes = set() - for part in suite.groups: - schemes.update([x.name for x in part.schemes()]) - # end for - # Write out the list - API.write_var_set_loop(ofile, 'scheme_list', schemes, 3) - else_str = 'else ' - # end for - ofile.write("else", 2) - emsg = "write({errmsg}, '(3a)')".format(errmsg=errmsg_name) - emsg += "'No suite named ', trim(suite_name), ' found'" - ofile.write(emsg, 3) - ofile.write("{errcode} = 1".format(errcode=errcode_name), 3) - ofile.write("end if", 2) - ofile.write("end subroutine {}".format(API.__schemes_fname), 1) - - def write_inspection_routines(self, ofile): - """Write the list_suites and list_suite_parts subroutines""" - errmsg_name, errcode_name = self.get_errinfo_names(base_only=True) - ofile.write("subroutine {}(suites)".format(API.__suite_fname), 1) - nsuites = len(self.suites) - oline = "character(len=*), allocatable, intent(out) :: suites(:)" - ofile.write(oline, 2) - ofile.write("\nallocate(suites({}))".format(nsuites), 2) - for ind, suite in enumerate(self.suites): - ofile.write("suites({}) = '{}'".format(ind+1, suite.name), 2) - # end for - ofile.write("end subroutine {}".format(API.__suite_fname), 1) - ofile.blank_line() - # Write out the suite part list subroutine - self.write_suite_part_list_sub(ofile, errmsg_name, errcode_name) - # Write out the suite required variable subroutine - self.write_req_vars_sub(ofile, errmsg_name, errcode_name) - # Write out the suite scheme list subroutine - self.write_suite_schemes_sub(ofile, errmsg_name, errcode_name) - - @property - def module(self): - """Return the module name of the API.""" - return self.__module - - @property - def host_model(self): - """Return the host model which will use this API.""" - return self.__host - - @property - def suite_name_var(self): - "Return the name of the variable specifying the suite to run" - return self.__suite_name - - @property - def suite_part_var(self): - "Return the name of the variable specifying the suite group to run" - return self.__suite_part - - @property - def suites(self): - "Return the list of this API's suites" - return self.__suites - -############################################################################### -if __name__ == "__main__": - try: - # First, run doctest - # pylint: disable=ungrouped-imports - import doctest - import sys - # pylint: enable=ungrouped-imports - fail, _ = doctest.testmod() - # Goal: Replace this test with a suite from unit tests - FRAME_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - TEMP_SUITE = os.path.join(FRAME_ROOT, 'test', 'capgen_test', - 'temp_suite.xml') - if os.path.exists(TEMP_SUITE): - _ = Suite(TEMP_SUITE, VarDictionary('temp_suite', - _API_DUMMY_RUN_ENV), - _API_DUMMY_RUN_ENV) - else: - print("Cannot find test file, '{}', skipping test".format(TEMP_SUITE)) - # end if - sys.exit(fail) - except CCPPError as suite_error: - print("{}".format(suite_error)) - sys.exit(fail) - # end try -# end if diff --git a/scripts/ccpp_track_variables.py b/scripts/ccpp_track_variables.py deleted file mode 100755 index 6d3f6d0b..00000000 --- a/scripts/ccpp_track_variables.py +++ /dev/null @@ -1,218 +0,0 @@ -#!/usr/bin/env python3 - -# Standard modules -import os -import argparse -import logging -import glob - -# CCPP framework imports -from metadata_table import find_scheme_names, parse_metadata_file -from ccpp_prebuild import import_config, gather_variable_definitions -from mkstatic import Suite -from common import lowercase_keys -from parse_checkers import registered_fortran_ddt_names -from parse_tools import init_log, set_log_level -from framework_env import CCPPFrameworkEnv - -############################################################################### -# Set up the command line argument parser and other global variables # -############################################################################### - -############################################################################### -# Functions and subroutines # -############################################################################### - -def parse_arguments(): - """Parse command line arguments.""" - parser = argparse.ArgumentParser() - parser.add_argument('-s', '--sdf', help='suite definition file to parse', required=True) - parser.add_argument('-m', '--metadata_path', - help='path to CCPP scheme metadata files', required=True) - parser.add_argument('-c', '--config', - help='path to CCPP prebuild configuration file', required=True) - parser.add_argument('-v', '--variable', help='variable to track through CCPP suite', - required=True) - parser.add_argument('--debug', action='store_true', help='enable debugging output', - default=False) - - args = parser.parse_args() - - return args - -def setup_logging(debug): - """Sets up the logging module and logging level.""" - - #Use capgen logging tools - logger = init_log('ccpp_track_variables') - - if debug: - set_log_level(logger, logging.DEBUG) - logger.info('Logging level set to DEBUG') - else: - set_log_level(logger, logging.WARNING) - return logger - -def parse_suite(sdf, run_env): - """Reads the provided sdf, parses into a Suite data structure, including the "call tree": - the ordered list of schemes for the suite specified by the provided sdf""" - run_env.logger.info(f'Reading sdf {sdf} and populating Suite object') - suite = Suite(sdf_name=sdf) - success = suite.parse(make_call_tree=True) - if not success: - raise Exception(f'Parsing suite definition file {sdf} failed.') - run_env.logger.info(f'Successfully read sdf {suite.sdf_name}') - return suite - -def create_metadata_filename_dict(metapath): - """Given a path, read all .meta files in that directory and add them to a dictionary: the keys - are the name of the scheme, and the values are the filename of the .meta file associated - with that scheme""" - - metadata_dict = {} - scheme_filenames = glob.glob(os.path.join(metapath, "*.meta"), recursive=True) - if not scheme_filenames: - raise Exception(f'No files found in {metapath} with ".meta" extension') - - for scheme_fn in scheme_filenames: - schemes = find_scheme_names(scheme_fn) - # The above returns a list of schemes in each filename, but - # we want a dictionary of schemes associated with filenames: - for scheme in schemes: - metadata_dict[scheme.lower()] = scheme_fn - - return metadata_dict - - -def create_var_graph(suite, var, config, metapath, run_env): - """Given a suite, variable name, a 'config' dictionary, and a path to .meta files: - 1. Creates a dictionary associating schemes with their .meta files - 2. Loops through the call tree of the provided suite by group - 3. For each scheme, reads .meta file for said scheme, checks for variable within that - scheme, and if it exists, adds an entry to a list of tuples for the corresponding - group, where each tuple includes the name of the scheme and the intent of the variable - within that scheme""" - - # Create a list of tuples for each group that will hold the in/out information for each scheme - var_graph={} - var_graph_empty = True - - run_env.logger.debug(f"reading .meta files in path:\n {metapath}") - metadata_dict=create_metadata_filename_dict(metapath) - - # Loop through call tree, find matching filename for scheme via dictionary schemes_in_files, - # then parse that metadata file to find variable info - partial_matches = {} - for group in suite.call_tree: - run_env.logger.debug(f"for group {group} ") - # Create list of tuples that will hold the in/out information for each scheme in this group - var_graph[group] = [] - for scheme in suite.call_tree[group]: - run_env.logger.debug(f"reading meta file for scheme {scheme} ") - - if scheme in metadata_dict: - scheme_filename = metadata_dict[scheme] - else: - raise Exception(f"Error, scheme '{scheme}' from suite '{suite.sdf_name}' " - f"not found in metadata files in {metapath}") - - run_env.logger.debug(f"reading metadata file {scheme_filename} for scheme {scheme}") - - new_metadata_headers = parse_metadata_file(scheme_filename, - known_ddts=registered_fortran_ddt_names(), - run_env=run_env) - for scheme_metadata in new_metadata_headers: - if scheme_metadata.table_name != scheme: - # Some metadata files contain information for multiple schemes, - # need to make sure we only read the relevant section - continue - for section in scheme_metadata.sections(): - found_var = [] - intent = '' - for scheme_var in section.variable_list(): - exact_match = False - if var == scheme_var.get_prop_value('standard_name'): - run_env.logger.debug(f"Found variable {var} in scheme {section.title}") - found_var=var - exact_match = True - intent = scheme_var.get_prop_value('intent') - break - scheme_var_standard_name = scheme_var.get_prop_value('standard_name') - if scheme_var_standard_name.find(var) != -1: - run_env.logger.debug(f"{var} matches {scheme_var_standard_name}") - found_var.append(scheme_var_standard_name) - if not found_var: - run_env.logger.debug(f"Did not find variable {var} in scheme {section.title}") - elif exact_match: - run_env.logger.debug(f"Exact match found for variable {var} in scheme " - f"{section.title}, intent {intent}") - var_graph[group].append((section.title,intent)) - var_graph_empty = False - else: - run_env.logger.debug(f"Found inexact matches for variable(s) {var} " - f"in scheme {section.title}:\n{found_var}") - partial_matches[section.title] = found_var - - - if not var_graph_empty: - success = True - run_env.logger.debug(f"Successfully generated variable graph for sdf {suite.sdf_name}\n") - else: - success = False - run_env.logger.error(f"Variable {var} not found in any suites for sdf {suite.sdf_name}\n") - if partial_matches: - print("Did find partial matches that may be of interest:\n") - for key in partial_matches: - print(f"In {key} found variable(s) {partial_matches[key]}") - - return (success,var_graph) - -def track_variables(sdf,metadata_path,config,variable,debug): - """Main routine that traverses a CCPP suite and outputs the list of schemes that use given - variable, broken down by group - - Args: - sdf (str) : The full path of the suite definition file to parse - metadata_path (str) : path to CCPP scheme metadata files - config (str) : path to CCPP prebuild configuration file - variable (str) : variable to track through CCPP suite - debug (bool) : Enable extra output for debugging - - Returns: - None -""" - - logger = setup_logging(debug) - - #Use new capgen class CCPPFrameworkEnv - run_env = CCPPFrameworkEnv(logger, host_files="", scheme_files="", suites="") - - suite = parse_suite(sdf,run_env) - - (success, config) = import_config(config, None) - if not success: - raise Exception('Call to import_config failed.') - - # Variables defined by the host model; this call is necessary because it converts some old - # metadata formats so they can be used later in the script - (success, _, _) = gather_variable_definitions(config['variable_definition_files'], - config['typedefs_new_metadata']) - if not success: - raise Exception('Call to gather_variable_definitions failed.') - - (success, var_graph) = create_var_graph(suite, variable, config, metadata_path, run_env) - if success: - print(f"For suite {suite.sdf_name}, the following schemes (in order for each group) " - f"use the variable {variable}:") - for group in var_graph: - if var_graph[group]: - print(f"In group {group}") - for entry in var_graph[group]: - print(f" {entry[0]} (intent {entry[1]})") - - -if __name__ == '__main__': - - args = parse_arguments() - - track_variables(args.sdf,args.metadata_path,args.config,args.variable,args.debug) diff --git a/scripts/code_block.py b/scripts/code_block.py deleted file mode 100644 index ccd3f209..00000000 --- a/scripts/code_block.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python3 -# - -"""Class and methods to create a code block which can then be written -to a file.""" - -# Python library imports -import re -# CCPP framework imports -from parse_tools import ParseContext, ParseSource, context_string -from parse_tools import ParseInternalError - -class CodeBlock(object): - """Class to store a block of code and a method to write it to a file - >>> CodeBlock([]) #doctest: +ELLIPSIS - - >>> CodeBlock(['hi mom']) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: Each element of must contain exactly two items, a code string and a relative indent - >>> CodeBlock([('hi mom')]) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: Each element of must contain exactly two items, a code string and a relative indent - >>> CodeBlock([('hi mom', 'x')]) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: Each element of must contain exactly two items, a code string and a relative indent - >>> CodeBlock([('hi mom', 1)]) #doctest: +ELLIPSIS - - >>> from fortran_tools import FortranWriter - >>> outfile_name = "__code_block_temp.F90" - >>> outfile = FortranWriter(outfile_name, 'w', 'test file', 'test_mod') - >>> CodeBlock([('hi mom', 1)]).write(outfile, 1, {}) - - >>> CodeBlock([('hi {greet} mom', 1)]).write(outfile, 1, {}) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: 'greet' missing from - >>> CodeBlock([('hi {{greet}} mom', 1)]).write(outfile, 1, {}) - >>> CodeBlock([('{greet} there mom', 1)]).write(outfile, 1, {'greet':'hi'}) - >>> outfile.__exit__() - False - >>> import os - >>> os.remove(outfile_name) - """ - - __var_re = re.compile(r"[{][ ]*([A-Za-z][A-Za-z0-9_]*)[ ]*[}]") - - __fmt_msg = ('Each element of must contain exactly two ' - 'items, a code string and a relative indent') - - def __init__(self, code_list): - """Initialize object with a list of statements. - Capture and store all variables required for output. - Each statement is a tuple consisting of a string and an indent level. - Non-negative indents will be added to a current indent at write time - while negative indents are written with no indentation. - """ - self.__code_block = code_list - self.__write_vars = list() - for line in self.__code_block: - if len(line) != 2: - raise ParseInternalError(CodeBlock.__fmt_msg.format(code_list)) - # end if - stmt = line[0] - if not isinstance(stmt, str): - raise ParseInternalError(CodeBlock.__fmt_msg.format(code_list)) - # end if - if not isinstance(line[1], int): - raise ParseInternalError(CodeBlock.__fmt_msg.format(code_list)) - # end if - beg = 0 - end = len(stmt) - while beg < end: - # Ignore double curly braces - open_double_curly = stmt.find('{{', beg) - close_double_curly = stmt.find('}}', - max(open_double_curly, beg)) - if 0 <= open_double_curly < close_double_curly: - beg = close_double_curly + 2 - else: - match = CodeBlock.__var_re.search(stmt[beg:]) - if match: - self.__write_vars.append(match.group(1)) - beg = stmt.index('}', beg) + 1 - else: - beg = end + 1 - # end if - # end if - # end while - # end for - - def write(self, outfile, indent_level, var_dict): - """Write this object's code block to using - as a basic offset. - Format each line using the variables from . - It is an error for to not contain any variable - indicated in the code block.""" - - for line in self.__code_block: - stmt = line[0] - if indent_level >= 0: - indent = indent_level + line[1] - else: - indent = 0 - # end if - # Check that contains all required items - errmsg = '' - sep = '' - for var in self.__write_vars: - if var not in var_dict: - errmsg += "'{}' missing from ".format(sep, var) - sep = '\n' - # end if - # end for - if errmsg: - raise ParseInternalError(errmsg) - # end if - outfile.write(stmt.format(**var_dict), indent) - # end for - -############################################################################### diff --git a/scripts/common.py b/scripts/common.py deleted file mode 100755 index 234e2521..00000000 --- a/scripts/common.py +++ /dev/null @@ -1,233 +0,0 @@ -#!/usr/bin/env python3 - -from collections import OrderedDict -import keyword -import logging -import os -import re -import subprocess -import sys - -# This dictionary contains short names for the different CCPP stages, -# because Fortran does not allow subroutine names with more than 63 characters -# Important: 'timestep_init' and 'timestep_finalize' need to come first so that -# a pattern match won't pick "init" for a CCPP subroutine name "xyz_timestep_init" -CCPP_STAGES = OrderedDict() -CCPP_STAGES['timestep_init'] = 'tsinit' -CCPP_STAGES['timestep_finalize'] = 'tsfinal' -CCPP_STAGES['init'] = 'init' -CCPP_STAGES['run'] = 'run' -CCPP_STAGES['finalize'] = 'final' - -CCPP_T_INSTANCE_VARIABLE = 'ccpp_t_instance' -CCPP_CONSTANT_ONE = 'ccpp_constant_one' -CCPP_ERROR_CODE_VARIABLE = 'ccpp_error_code' -CCPP_ERROR_MSG_VARIABLE = 'ccpp_error_message' -CCPP_LOOP_COUNTER = 'ccpp_loop_counter' -CCPP_LOOP_EXTENT = 'ccpp_loop_extent' -CCPP_BLOCK_NUMBER = 'ccpp_block_number' -CCPP_BLOCK_COUNT = 'ccpp_block_count' -CCPP_BLOCK_SIZES = 'ccpp_block_sizes' -CCPP_THREAD_NUMBER = 'ccpp_thread_number' -CCPP_THREAD_COUNT = 'ccpp_thread_count' - -CCPP_CHUNK_EXTENT = 'ccpp_chunk_extent' -CCPP_HORIZONTAL_LOOP_BEGIN = 'horizontal_loop_begin' -CCPP_HORIZONTAL_LOOP_END = 'horizontal_loop_end' - -CCPP_HORIZONTAL_LOOP_EXTENT = 'horizontal_loop_extent' -CCPP_HORIZONTAL_DIMENSION = 'horizontal_dimension' - -FORTRAN_CONDITIONAL_REGEX_WORDS = [' ', '(', ')', '==', '/=', '<=', '>=', '<', '>', '.eqv.', '.neqv.', - '.true.', '.false.', '.lt.', '.le.', '.eq.', '.ge.', '.gt.', '.ne.', - '.not.', '.and.', '.or.', '.xor.'] -FORTRAN_CONDITIONAL_REGEX = re.compile(r"[\w']+|" + "|".join([word.replace('(','\(').replace(')', '\)') for word in FORTRAN_CONDITIONAL_REGEX_WORDS])) - -CCPP_TYPE = 'ccpp_t' - -# SCRIPTDIR is the directory where ccpp_prebuild.py and its Python modules are located -SCRIPTDIR = os.path.abspath(os.path.dirname(__file__)) - -# SRCDIR is the directory where the CCPP framework source code (C, Fortran) is located -SRCDIR = os.path.abspath(os.path.join(SCRIPTDIR, os.pardir, 'src')) - -# Definition of variables (metadata tables) that are provided by CCPP -CCPP_INTERNAL_VARIABLE_DEFINITON_FILE = os.path.join(SRCDIR, 'ccpp_types.F90') - -# List of internal variables provided by the CCPP -CCPP_INTERNAL_VARIABLES = { - CCPP_ERROR_CODE_VARIABLE : 'cdata%errflg', - CCPP_ERROR_MSG_VARIABLE : 'cdata%errmsg', - CCPP_LOOP_COUNTER : 'cdata%loop_cnt', - CCPP_BLOCK_NUMBER : 'cdata%blk_no', - CCPP_THREAD_NUMBER : 'cdata%thrd_no', - CCPP_THREAD_COUNT : 'cdata%thrd_cnt', - } - -STANDARD_CHARACTER_TYPE = 'character' -STANDARD_INTEGER_TYPE = 'integer' -STANDARD_VARIABLE_TYPES = [ STANDARD_CHARACTER_TYPE, STANDARD_INTEGER_TYPE, 'logical', 'real' ] - -# For static build -CCPP_STATIC_API_MODULE = 'ccpp_static_api' -CCPP_STATIC_SUBROUTINE_NAME = 'ccpp_physics_{stage}' - -# Filename pattern for suite definition files -SUITE_DEFINITION_FILENAME_PATTERN = re.compile('^(.*)\.xml$') - -# Maximum number of concurrent CCPP instances per MPI task -CCPP_NUM_INSTANCES = 200 - -def split_var_name_and_array_reference(var_name): - """Split an expression like foo(:,a,1:ddt%ngas) - into components foo and (:,a,1:ddt%ngas).""" - actual_var_name = None - array_reference = None - # Search for first pair of parentheses from the end of the string - parentheses = 0 - i = len(var_name)-1 - while i>=0: - if var_name[i] == ')': - parentheses += 1 - elif var_name[i] == '(': - parentheses -= 1 - if parentheses == 0: - actual_var_name = var_name[:i] - array_reference = var_name[i:] - break - i -= 1 - return (actual_var_name, array_reference) - -def encode_container(*args): - """Encodes a container, i.e. the location of a metadata table for CCPP. - Currently, there are three possibilities with different numbers of input - arguments: module, module+typedef, module+scheme+subroutine. Convert all - names to lowercase to support the new case-insensitive capgen parser.""" - if len(args)==3: - container = 'MODULE_{0} SCHEME_{1} SUBROUTINE_{2}'.format(*[arg.lower() for arg in args]) - elif len(args)==2: - container = 'MODULE_{0} TYPE_{1}'.format(*[arg.lower() for arg in args]) - elif len(args)==1: - container = 'MODULE_{0}'.format(*[arg.lower() for arg in args]) - else: - raise Exception("encode_container not implemented for {0} arguments".format(len(args))) - return container - -def decode_container(container): - """Decodes a container, i.e. the description of a location of a metadata table - for CCPP. Currently, there are three possibilities with different numbers of - input arguments: module, module+typedef, module+scheme+subroutine.""" - items = container.split(' ') - if not len(items) in [1, 2, 3]: - raise Exception("decode_container not implemented for {0} items".format(len(items))) - for i in range(len(items)): - items[i] = items[i][:items[i].find('_')] + ' ' + items[i][items[i].find('_')+1:] - return ' '.join(items) - -def decode_container_as_dict(container): - """Decodes a container, i.e. the description of a location of a metadata table - for CCPP. Currently, there are three possibilities with different numbers of - input arguments: module, module+typedef, module+scheme+subroutine. Return - a dictionary with possible keys MODULE, TYPE, SCHEME, SUBROUTINE.""" - items = container.split(' ') - if not len(items) in [1, 2, 3]: - raise Exception("decode_container not implemented for {0} items".format(len(items))) - itemsdict = {} - for i in range(len(items)): - key, value = (items[i][:items[i].find('_')], items[i][items[i].find('_')+1:]) - itemsdict[key] = value - return itemsdict - -def escape_tex(text): - """Substitutes characters for generating LaTeX sources files from Python.""" - return text.replace( - '%', '\%').replace( - '_', '\_').replace( - '&', '\&') - -def isstring(s): - """Return true if a variable is a string""" - return isinstance(s, str) - -def insert_plus_sign_for_positive_exponents(string): - """Parse a string (a unit string) and insert plus (+) signs - for positive exponents where needed""" - # Break up the string by spaces - items = string.split() - # Identify units with positive exponents - # without a plus sign (m2 instead of m+2). - pattern = re.compile(r"([a-zA-Z]+)([0-9]+)") - for index, item in enumerate(items): - match = pattern.match(item) - if match: - items[index] = "+".join(match.groups()) - # Recombine items to string - return " ".join(items) - -def string_to_python_identifier(string): - """Replaces forbidden characters in strings with standard substitutions - so that the result is a valid Python object (variable, function) name. - At this point, it only converts characters found in the units attributes. - A check for allowed characters in Python v2 catches missing conversions.""" - # Replace whitespaces with underscores - string = string.replace(" ","_") - # Replace decimal points with _p_ - string = string.replace(".","_p_") - # Replace dashes and minus sign with _minus_ - string = string.replace("-","_minus_") - # Replace plus signs with _plus_ - string = string.replace("+","_plus_") - # "1" is a valid unit - if string == "1": - string = "one" - # Test that the resulting string is a valid Python identifier - if re.match("[_A-Za-z][_a-zA-Z0-9]*$", string) and not keyword.iskeyword(string): - return string - else: - raise Exception("Resulting string '{0}' is not a valid Python identifier".format(string)) - -# New utilities added 2025/07/25 for dealing with case-insensitivity changes from capgen. Used to convert XML data read with the xml library and other ccpp_prebuild dictionaries - -def lowercase_keys_and_values(d): - """Recursively convert all keys and values in a regular dictionary to lowercase""" - if isinstance(d, dict): - return { - (k.lower() if isinstance(k, str) else k): - lowercase_keys_and_values(v) - for k, v in d.items() - } - elif isinstance(d, list): - return [lowercase_keys_and_values(item) for item in d] - elif isinstance(d, str): - return d.lower() - else: - return d - -def lowercase_keys(d): - """Recursively convert all keys in an OrderedDict to lowercase""" - if isinstance(d, OrderedDict): - new_dict = OrderedDict() - for k, v in d.items(): - new_key = k.lower() if isinstance(k, str) else k - new_dict[new_key] = lowercase_keys(v) - return new_dict - elif isinstance(d, list): - return [lowercase_keys(item) for item in d] - else: - return d - -def lowercase_xml(element): - """Recursively convert XML elements to lowercase""" - # Lowercase the tag name - element.tag = element.tag.lower() - # Lowercase the text content, if it exists - if element.text: - element.text = element.text.lower() - if element.tail: - element.tail = element.tail.lower() - # Lowercase attribute keys and values - element.attrib = {k.lower(): v.lower() for k, v in element.attrib.items()} - # Recurse into child elements - for i in range(len(element)): - element[i] = lowercase_xml(element[i]) - return element diff --git a/scripts/constituents.py b/scripts/constituents.py deleted file mode 100644 index e7e182c0..00000000 --- a/scripts/constituents.py +++ /dev/null @@ -1,798 +0,0 @@ -#!/usr/bin/env python3 - -""" -Class and supporting code to hold all information on CCPP constituent -variables. A constituent variable is defined and maintained by the CCPP -Framework instead of the host model. -The ConstituentVarDict class contains methods to generate the necessary code -to implement this support. -""" - -# CCPP framework imports -from parse_tools import ParseInternalError -from metavar import VarDictionary - -######################################################################## - -CONST_DDT_NAME = "ccpp_model_constituents_t" -CONST_DDT_MOD = "ccpp_constituent_prop_mod" -CONST_PROP_TYPE = "ccpp_constituent_properties_t" -CONST_PROP_PTR_TYPE = "ccpp_constituent_prop_ptr_t" -CONST_OBJ_STDNAME = "ccpp_model_constituents_object" - -######################################################################## - -class ConstituentVarDict(VarDictionary): - """A class to hold all the constituent variables for a CCPP Suite. - Also contains methods to generate the necessary code for runtime - allocation and support for these variables. - """ - - __const_prop_array_name = "ccpp_constituents" - __const_prop_init_name = "ccpp_constituents_initialized" - __const_prop_init_consts = "ccpp_create_constituent_array" - __constituent_type = "suite" - - def __init__(self, name, parent_dict, run_env, variables=None): - """Create a specialized VarDictionary for constituents. - The main difference is functionality to allocate and support - these variables with special functions for the host model. - The main reason for a separate dictionary is that these are not - proper Suite variables but will belong to the host model at run time. - The feature of the VarDictionary class is required - because this dictionary must be connected to a host model. - """ - self.__run_env = run_env - super().__init__(name, run_env, - variables=variables, parent_dict=parent_dict) - - def find_variable(self, standard_name=None, source_var=None, - any_scope=True, clone=None, - search_call_list=False, loop_subst=False): - """Attempt to return the variable matching . - if is None, the standard name from is used. - It is an error to pass both and if - the standard name of is not the same as . - If is True, search parent scopes if not in current scope. - Note: Unlike the version of this method, the case for - CCPP_CONSTANT_VARS is not handled -- it should have been handled - by a lower level. - If the variable is not found but is a constituent variable type, - create the variable in this dictionary - Note that although the argument is accepted for consistency, - cloning is not handled at this level. - If the variable is not found and is not a constituent - variable, return None. - """ - if standard_name is None: - if source_var is None: - emsg = "One of or must be passed." - raise ParseInternalError(emsg) - # end if - standard_name = source_var.get_prop_value('standard_name') - elif source_var is not None: - stest = source_var.get_prop_value('standard_name') - if stest != standard_name: - emsg = ("Only one of or may " + - "be passed.") - raise ParseInternalError(emsg) - # end if - # end if - if standard_name in self: - var = self[standard_name] - elif any_scope and (self.parent is not None): - srch_clist = search_call_list - var = self.parent.find_variable(standard_name=standard_name, - source_var=source_var, - any_scope=any_scope, clone=None, - search_call_list=srch_clist, - loop_subst=loop_subst) - else: - var = None - # end if - if (var is None) and source_var and source_var.is_constituent(): - # If we did not find the variable and it is a constituent type, - # add a clone of to our dictionary. - # First, maybe do a loop substitution - dims = source_var.get_dimensions() - newdims = list() - for dim in dims: - dstdnames = dim.split(':') - new_dnames = list() - for dstdname in dstdnames: - if dstdname == 'horizontal_loop_extent': - new_dnames.append('horizontal_dimension') - elif dstdname == 'horizontal_loop_end': - new_dnames.append('horizontal_dimension') - elif dstdname == 'horizontal_loop_begin': - new_dnames.append('ccpp_constant_one') - else: - new_dnames.append(dstdname) - # end if - # end for - newdims.append(':'.join(new_dnames)) - # end for - var = source_var.clone({'dimensions' : newdims}, remove_intent=True, - source_type=self.__constituent_type) - self.add_variable(var, self.__run_env) - return var - - @staticmethod - def __init_err_var(evar, outfile, indent): - """If is a known error variable, generate the code to - initialize it as an output variable. - If unknown, simply ignore. - """ - stdname = evar.get_prop_value('standard_name') - if stdname == 'ccpp_error_message': - lname = evar.get_prop_value('local_name') - outfile.write(f"{lname} = ''", indent) - elif stdname == 'ccpp_error_code': - lname = evar.get_prop_value('local_name') - outfile.write(f"{lname} = 0", indent) - # end if (no else, just ignore) - - def declare_public_interfaces(self, outfile, indent): - """Declare the public constituent interfaces. - Declarations are written to at indent, .""" - outfile.write("! Public interfaces for handling constituents", indent) - outfile.write("! Return the number of constituents for this suite", - indent) - outfile.write(f"public :: {self.num_consts_funcname()}", indent) - outfile.write("! Return the name of a constituent", indent) - outfile.write(f"public :: {self.const_name_subname()}", indent) - outfile.write("! Copy the data for a constituent", indent) - outfile.write(f"public :: {self.copy_const_subname()}", indent) - - def declare_private_data(self, outfile, indent): - """Declare private suite module variables and interfaces - to with indent, .""" - outfile.write("! Private constituent module data", indent) - if self: - stmt = f"type({CONST_PROP_TYPE}), private, allocatable :: {self.constituent_prop_array_name()}(:)" - outfile.write(stmt, indent) - # end if - stmt = f"logical, private :: {self.constituent_prop_init_name()} = .false." - outfile.write(stmt, indent) - outfile.write("! Private interface for constituents", indent) - stmt = f"private :: {self.constituent_prop_init_consts()}" - outfile.write(stmt, indent) - - @classmethod - def __errcode_names(cls, err_vars): - """Return the ( ) where is the local name - for ccpp_error_code in and is the local name for - ccpp_error_message in . - if either variable is not found in , return None.""" - errcode = None - errmsg = None - for evar in err_vars: - stdname = evar.get_prop_value('standard_name') - if stdname == 'ccpp_error_code': - errcode = evar.get_prop_value('local_name') - elif stdname == 'ccpp_error_message': - errmsg = evar.get_prop_value('local_name') - else: - emsg = f"Bad errcode variable, '{stdname}'" - raise ParseInternalError(emsg) - # end if - # end for - if (not errcode) or (not errmsg): - raise ParseInternalError("Unsupported error scheme") - # end if - return errcode, errmsg - - @staticmethod - def __errcode_callstr(errcode_name, errmsg_name, suite): - """Create and return the error code calling string for . - is the calling routine's ccpp_error_code variable name. - is the calling routine's ccpp_error_message variable name. - """ - err_vars = suite.find_error_variables(any_scope=True, clone_as_out=True) - errcode, errmsg = ConstituentVarDict.__errcode_names(err_vars) - errvar_str = f"{errcode}={errcode_name}, {errmsg}={errmsg_name}" - return errvar_str - - def _write_init_check(self, outfile, indent, suite_name, - err_vars, use_errcode): - """Write a check to to make sure the constituent properties - are initialized. Write code to initialize the error variables and/or - set them to error values.""" - outfile.write('', 0) - if use_errcode: - errcode, errmsg = self.__errcode_names(err_vars) - outfile.write(f"{errcode} = 0", indent+1) - outfile.write(f"{errmsg} = ''", indent+1) - else: - raise ParseInternalError("Alternative to errcode not implemented") - # end if - outfile.write("! Make sure that our constituent array is initialized", - indent+1) - stmt = f"if (.not. {self.constituent_prop_init_name()}) then" - outfile.write(stmt, indent+1) - if use_errcode: - outfile.write(f"{errcode} = 1", indent+2) - stmt = f'{errmsg} = "constituent properties not ' - stmt += f'initialized for suite, {suite_name}"' - outfile.write(stmt, indent+2) - outfile.write("end if", indent+1) - # end if (no else until an alternative error mechanism supported) - - def _write_index_check(self, outfile, indent, suite_name, - err_vars, use_errcode): - """Write a check to to make sure the "index" input - is in bounds. Write code to set error variables if index is - out of bounds.""" - if use_errcode: - errcode, errmsg = self.__errcode_names(err_vars) - if self: - outfile.write("if (index < 1) then", indent+1) - outfile.write(f"{errcode} = 1", indent+2) - stmt = f"write({errmsg}, '(a,i0,a)') 'ERROR: index (',index,') " - stmt += "too small, must be >= 1'" - outfile.write(stmt, indent+2) - stmt = f"else if (index > SIZE({self.constituent_prop_array_name()})) then" - outfile.write(stmt, indent+1) - outfile.write(f"{errcode} = 1", indent+2) - stmt = f"write({errmsg}, '(2(a,i0))') 'ERROR: index (',index,') " - stmt += f"too large, must be <= ', SIZE({self.constituent_prop_array_name()})" - outfile.write(stmt, indent+2) - outfile.write("end if", indent+1) - else: - outfile.write(f"{errcode} = 1", indent+1) - stmt = f"write({errmsg}, '(a,i0,a)') 'ERROR: {self.name}, " - stmt += "has no constituents'" - outfile.write(stmt, indent+1) - # end if - else: - raise ParseInternalError("Alternative to errcode not implemented") - # end if - - def write_constituent_routines(self, outfile, indent, suite_name, err_vars): - """Write the subroutine that, when called allocates and defines the - suite-cap module variable describing the constituent species for - this suite. - Code is written to starting at indent, .""" - # Format our error variables - errvar_names = {x.get_prop_value('standard_name') : - x.get_prop_value('local_name') for x in err_vars} - errcode_snames = ('ccpp_error_code', 'ccpp_error_message') - use_errcode = all([x.get_prop_value('standard_name') in errcode_snames - for x in err_vars]) - errvar_alist = ", ".join([x for x in errvar_names.values()]) - errvar_alist2 = f", {errvar_alist}" if errvar_alist else "" - call_vnames = {'ccpp_error_code' : 'errcode', - 'ccpp_error_message' : 'errmsg'} - errvar_call = ", ".join([f"{call_vnames[x]}={errvar_names[x]}" for x in errcode_snames]) - errvar_call2 = f", {errvar_call}" if errvar_call else "" - local_call = ", ".join([f"{errvar_names[x]}={errvar_names[x]}" for x in errcode_snames]) - # Allocate and define constituents - stmt = f"subroutine {self.constituent_prop_init_consts()}({errvar_alist})" - outfile.write(stmt, indent) - outfile.write("! Allocate and fill the constituent property array", - indent + 1) - outfile.write("! for this suite", indent+1) - outfile.write("! Dummy arguments", indent+1) - for evar in err_vars: - evar.write_def(outfile, indent+1, self, dummy=True) - # end for - # Figure out how many unique (non-tendency) constituent variables we have - const_num = 0 - for std_name, _ in self.items(): - if not std_name.startswith('tendency_of_'): - const_num += 1 - # end if - # end for - if self: - outfile.write("! Local variables", indent+1) - outfile.write("integer :: index", indent+1) - stmt = f"allocate({self.constituent_prop_array_name()}({const_num}))" - outfile.write(stmt, indent+1) - outfile.write("index = 0", indent+1) - # end if - for evar in err_vars: - self.__init_err_var(evar, outfile, indent+1) - # end for - for std_name, var in self.items(): - if std_name.startswith('tendency_of_'): - # Skip tendency variables - continue - # end if - outfile.write("index = index + 1", indent+1) - long_name = var.get_prop_value('long_name') - diag_name = var.get_prop_value('diagnostic_name') - units = var.get_prop_value('units') - dims = var.get_dim_stdnames() - default_value = var.get_prop_value('default_value') - if 'vertical_layer_dimension' in dims: - vertical_dim = 'vertical_layer_dimension' - elif 'vertical_interface_dimension' in dims: - vertical_dim = 'vertical_interface_dimension' - else: - vertical_dim = '' - # end if - advect_str = self.TF_string(var.get_prop_value('advected')) - init_args = [f'{std_name=}', f'{long_name=}', f'{diag_name=}', - f'{units=}', f'{vertical_dim=}', - f'advected={advect_str}', - f'errcode={errvar_names["ccpp_error_code"]}', - f'errmsg={errvar_names["ccpp_error_message"]}'] - if default_value is not None and default_value != '': - init_args.append(f'default_value={default_value}') - stmt = f'call {self.constituent_prop_array_name()}(index)%instantiate({", ".join(init_args)})' - outfile.write(f'if ({errvar_names["ccpp_error_code"]} == 0) then', indent+1) - outfile.write(stmt, indent+2) - outfile.write("end if", indent+1) - # end for - outfile.write(f"{self.constituent_prop_init_name()} = .true.", indent+1) - stmt = f"end subroutine {self.constituent_prop_init_consts()}" - outfile.write(stmt, indent) - outfile.write("", 0) - border = "="*72 - outfile.write(f"\n! {border}\n", 1) - # Return number of constituents - fname = self.num_consts_funcname() - outfile.write(f"integer function {fname}({errvar_alist})", indent) - outfile.write("! Return the number of constituents for this suite", - indent+1) - outfile.write("! Dummy arguments", indent+1) - for evar in err_vars: - evar.write_def(outfile, indent+1, self, dummy=True) - # end for - for evar in err_vars: - self.__init_err_var(evar, outfile, indent+1) - # end for - outfile.write("! Make sure that our constituent array is initialized", - indent+1) - stmt = f"if (.not. {self.constituent_prop_init_name()}) then" - outfile.write(stmt, indent+1) - outfile.write(f"call {self.constituent_prop_init_consts()}({local_call})", indent+2) - outfile.write("end if", indent+1) - outfile.write(f"{fname} = {const_num}", indent+1) - outfile.write(f"end function {fname}", indent) - outfile.write(f"\n! {border}\n", 1) - # Return the name of a constituent given an index - stmt = f"subroutine {self.const_name_subname()}(index, name_out{errvar_alist2})" - outfile.write(stmt, indent) - outfile.write("! Return the name of constituent, ", indent+1) - outfile.write("! Dummy arguments", indent+1) - outfile.write("integer, intent(in) :: index", indent+1) - outfile.write("character(len=*), intent(out) :: name_out", indent+1) - for evar in err_vars: - evar.write_def(outfile, indent+1, self, dummy=True) - # end for - self._write_init_check(outfile, indent, suite_name, - err_vars, use_errcode) - self._write_index_check(outfile, indent, suite_name, - err_vars, use_errcode) - if self: - init_args = ['std_name=name_out', - f'errcode={errvar_names["ccpp_error_code"]}', - f'errmsg={errvar_names["ccpp_error_message"]}'] - stmt = f"call {self.constituent_prop_array_name()}(index)%standard_name({', '.join(init_args)})" - outfile.write(stmt, indent+1) - # end if - outfile.write(f"end subroutine {self.const_name_subname()}", indent) - outfile.write(f"\n! {border}\n", 1) - # Copy a consitituent's properties - fname = self.copy_const_subname() - stmt = f"subroutine {fname}(index, cnst_out{errvar_alist2})" - outfile.write(stmt, indent) - outfile.write("! Copy the data for a constituent", indent+1) - outfile.write("! Dummy arguments", indent+1) - outfile.write("integer, intent(in) :: index", indent+1) - stmt = f"type({CONST_PROP_TYPE}), intent(out) :: cnst_out" - outfile.write(stmt, indent+1) - for evar in err_vars: - evar.write_def(outfile, indent+1, self, dummy=True) - # end for - self._write_init_check(outfile, indent, suite_name, - err_vars, use_errcode) - self._write_index_check(outfile, indent, suite_name, - err_vars, use_errcode) - if self: - stmt = f"cnst_out = {self.constituent_prop_array_name()}(index)" - outfile.write(stmt, indent+1) - # end if - outfile.write(f"end subroutine {fname}", indent) - - def constituent_module_name(self): - """Return the name of host model constituent module""" - if not ((self.parent is not None) and - hasattr(self.parent.parent, "constituent_module")): - emsg = "ConstituentVarDict parent not HostModel?" - emsg += f"\nparent is '{type_name(self.parent.parent)}'" - raise ParseInternalError(emsg) - # end if - return self.parent.parent.constituent_module - - def num_consts_funcname(self): - """Return the name of the function which returns the number of - constituents for this suite.""" - return f"{self.name}_num_consts" - - def const_name_subname(self): - """Return the name of the routine that returns a constituent's - standard name given an index""" - return f"{self.name}_const_name" - - def copy_const_subname(self): - """Return the name of the routine that returns a copy of a - constituent's metadata given an index""" - return f"{self.name}_copy_const" - - @staticmethod - def constituent_index_name(standard_name): - """Return the index name associated with """ - return f"index_of_{standard_name}" - - @staticmethod - def write_constituent_use_statements(cap, suite_list, indent): - """Write the suite use statements needed by the constituent - initialization routines.""" - maxmod = max([len(s.module) for s in suite_list]) - smod = len(CONST_DDT_MOD) - maxmod = max(maxmod, smod) - use_str = "use {},{} only: {}" - spc = ' '*(maxmod - smod) - cap.write(use_str.format(CONST_DDT_MOD, spc, CONST_PROP_TYPE), indent) - cap.write('! Suite constituent interfaces', indent) - for suite in suite_list: - const_dict = suite.constituent_dictionary() - smod = suite.module - spc = ' '*(maxmod - len(smod)) - fname = const_dict.num_consts_funcname() - cap.write(use_str.format(smod, spc, fname), indent) - fname = const_dict.const_name_subname() - cap.write(use_str.format(smod, spc, fname), indent) - fname = const_dict.copy_const_subname() - cap.write(use_str.format(smod, spc, fname), indent) - # end for - - @staticmethod - def write_host_routines(cap, host, reg_funcname, init_funcname, num_const_funcname, - query_const_funcname, copy_in_funcname, copy_out_funcname, cleanup_funcname, - const_obj_name, dyn_const_names, const_names_name, const_indices_name, - const_array_func, advect_array_func, prop_array_func, - const_index_func, suite_list, err_vars): - """Write out the host model routine which will - instantiate constituent fields for all the constituents in . - is a list of the host model's error variables. - Also write out the following routines: - : Initialize constituent data - : Number of constituents - : Check if standard name matches existing constituent - : Collect constituent fields for host - : Update constituent fields from host - : Return a pointer to the constituent array - : Return a pointer to the advected constituent array - : Return a pointer to the constituent properties array - : Return the index of a provided constituent name - Output is written to . - """ -# XXgoldyXX: v need to generalize host model error var type support - use_errcode = [x.get_prop_value('standard_name') in - ('ccpp_error_code' 'ccpp_error_message') - for x in err_vars] - if not use_errcode: - emsg = "Error object not supported for {}" - raise ParseInternalError(emsg(host.name)) - # end if - herrcode, herrmsg = ConstituentVarDict.__errcode_names(err_vars) - err_dummy_str = f"{herrcode}, {herrmsg}" - obj_err_callstr = f"errcode={herrcode}, errmsg={herrmsg}" -# XXgoldyXX: ^ need to generalize host model error var type support - # First up, the registration routine - substmt = f"subroutine {reg_funcname}" - args = "host_constituents " - stmt = f"{substmt}({args}, {err_dummy_str})" - cap.write(stmt, 1) - cap.comment("Create constituent object for suites in ", 2) - cap.blank_line() - ConstituentVarDict.write_constituent_use_statements(cap, suite_list, 2) - cap.blank_line() - cap.comment("Dummy arguments", 2) - cap.write(f"type({CONST_PROP_TYPE}), target, intent(in) :: " + \ - "host_constituents(:)", 2) - for evar in err_vars: - evar.write_def(cap, 2, host, dummy=True, - add_intent="out", extra_space=25) - # end for - cap.comment("Local variables", 2) - spc = ' '*37 - cap.write(f"integer{spc} :: num_suite_consts", 2) - cap.write(f"integer{spc} :: num_consts", 2) - cap.write(f"integer{spc} :: index, index_start", 2) - cap.write(f"integer{spc} :: field_ind", 2) - cap.write(f"type({CONST_PROP_TYPE}), pointer :: const_prop => NULL()", 2) - cap.blank_line() - cap.write(f"{herrcode} = 0", 2) - cap.write("num_consts = size(host_constituents, 1)", 2) - for suite in suite_list: - const_dict = suite.constituent_dictionary() - funcname = const_dict.num_consts_funcname() - cap.comment(f"Number of suite constants for {suite.name}", 2) - errvar_str = ConstituentVarDict.__errcode_callstr(herrcode, - herrmsg, suite) - cap.write(f"num_suite_consts = {funcname}({errvar_str})", 2) - cap.write(f"if ({herrcode} /= 0) then", 2) - cap.write("return", 3) - cap.write("end if", 2) - cap.write("num_consts = num_consts + num_suite_consts", 2) - # end for - cap.comment("Initialize constituent data and field object", 2) - stmt = f"call {const_obj_name}%initialize_table(num_consts)" - cap.write(stmt, 2) - # Register host model constituents - cap.comment("Add host model constituent metadata", 2) - cap.write("do index = 1, size(host_constituents, 1)", 2) - cap.write("const_prop => host_constituents(index)", 3) - stmt = f"call {const_obj_name}%new_field(const_prop, {obj_err_callstr})" - cap.write(stmt, 3) - cap.write("nullify(const_prop)", 3) - cap.write(f"if ({herrcode} /= 0) then", 3) - cap.write("return", 4) - cap.write("end if", 3) - cap.write("end do", 2) - cap.blank_line() - # Register dynamic constituents - cap.comment("Add dynamic constituent properties", 2) - for dyn_const in dyn_const_names: - cap.write(f"do index = 1, size({dyn_const}, 1)", 2) - cap.write(f"const_prop => {dyn_const}(index)", 3) - stmt = f"call {const_obj_name}%new_field(const_prop, {obj_err_callstr})" - cap.write(stmt, 3) - cap.write("nullify(const_prop)", 3) - cap.write(f"if ({herrcode} /= 0) then", 3) - cap.write("return", 4) - cap.write("end if", 3) - cap.write("end do", 2) - # end for - - # Register suite constituents - for suite in suite_list: - errvar_str = ConstituentVarDict.__errcode_callstr(herrcode, - herrmsg, suite) - cap.comment(f"Add {suite.name} constituent metadata", 2) - const_dict = suite.constituent_dictionary() - funcname = const_dict.num_consts_funcname() - cap.write(f"num_suite_consts = {funcname}({errvar_str})", 2) - cap.write(f"if ({herrcode} /= 0) then", 2) - cap.write("return", 3) - cap.write("end if", 2) - funcname = const_dict.copy_const_subname() - cap.write("do index = 1, num_suite_consts", 2) - cap.write(f"allocate(const_prop, stat={herrcode})", 3) - cap.write(f"if ({herrcode} /= 0) then", 3) - cap.write(f'{herrmsg} = "ERROR allocating const_prop"', 4) - cap.write("return", 4) - cap.write("end if", 3) - stmt = f"call {funcname}(index, const_prop, {errvar_str})" - cap.write(stmt, 3) - cap.write(f"if ({herrcode} /= 0) then", 3) - cap.write("return", 4) - cap.write("end if", 3) - stmt = f"call {const_obj_name}%new_field(const_prop, {obj_err_callstr})" - cap.write(stmt, 3) - cap.write("nullify(const_prop)", 3) - cap.write(f"if ({herrcode} /= 0) then", 3) - cap.write("return", 4) - cap.write("end if", 3) - cap.write("end do", 2) - cap.blank_line() - # end for - stmt = f"call {const_obj_name}%lock_table({obj_err_callstr})" - cap.write(stmt, 2) - cap.write(f"if ({herrcode} /= 0) then", 2) - cap.write("return", 3) - cap.write("end if", 2) - cap.comment("Set the index for each active constituent", 2) - cap.write(f"do index = 1, SIZE({const_indices_name})", 2) - stmt = f"call {const_obj_name}%const_index(field_ind, {const_names_name}(index), {obj_err_callstr})" - cap.write(stmt, 3) - cap.write(f"if ({herrcode} /= 0) then", 3) - cap.write("return", 4) - cap.write("end if", 3) - cap.write("if (field_ind > 0) then", 3) - cap.write(f"{const_indices_name}(index) = field_ind", 4) - cap.write("else", 3) - cap.write(f"{herrcode} = 1", 4) - stmt = f"{herrmsg} = 'No field index for '//trim({const_names_name}(index))" - cap.write(stmt, 4) - cap.write("return", 4) - cap.write("end if", 3) - cap.write("end do", 2) - cap.write(f"end {substmt}", 1) - # Write constituent_init routine - substmt = f"subroutine {init_funcname}" - cap.blank_line() - cap.write(f"{substmt}(ncols, num_layers, {err_dummy_str})", 1) - cap.comment("Initialize constituent data", 2) - cap.blank_line() - cap.write("use ccpp_scheme_utils, only: ccpp_initialize_constituent_ptr", 2) - cap.blank_line() - cap.comment("Dummy arguments", 2) - cap.write("integer, intent(in) :: ncols", 2) - cap.write("integer, intent(in) :: num_layers", 2) - for evar in err_vars: - evar.write_def(cap, 2, host, dummy=True, add_intent="out") - # end for evar - cap.blank_line() - call_str = f"call {const_obj_name}%lock_data(ncols, num_layers, {obj_err_callstr})" - cap.write(call_str, 2) - cap.write(f"call ccpp_initialize_constituent_ptr({const_obj_name})", 2) - cap.write(f"end {substmt}", 1) - # Write num_consts routine - substmt = f"subroutine {num_const_funcname}" - cap.blank_line() - cap.write(f"{substmt}(num_flds, advected, {err_dummy_str})", 1) - cap.comment("Return the number of constituent fields for this run", 2) - cap.blank_line() - cap.comment("Dummy arguments", 2) - cap.write("integer, intent(out) :: num_flds", 2) - cap.write("logical, optional, intent(in) :: advected", 2) - for evar in err_vars: - evar.write_def(cap, 2, host, dummy=True, add_intent="out") - # end for - cap.blank_line() - call_str = f"call {const_obj_name}%num_constituents(num_flds, advected=advected, {obj_err_callstr})" - cap.write(call_str, 2) - cap.write(f"end {substmt}", 1) - # Write query_consts routine - substmt = f"subroutine {query_const_funcname}" - cap.blank_line() - cap.write(f"{substmt}(var_name, constituent_exists, {err_dummy_str})", 1) - cap.comment(f"Return constituent_exists = true iff var_name appears in {host.name}_model_const_stdnames", 2) - cap.blank_line() - cap.write("character(len=*), intent(in) :: var_name", 2) - cap.write("logical, intent(out) :: constituent_exists", 2) - for evar in err_vars: - evar.write_def(cap, 2, host, dummy=True, add_intent="out") - # end for - cap.blank_line() - cap.write(f"{herrcode} = 0", 2) - cap.write(f"{herrmsg} = ''", 2) - cap.blank_line() - cap.write("constituent_exists = .false.", 2) - cap.write(f"if (any({host.name}_model_const_stdnames == var_name)) then", 2) - cap.write("constituent_exists = .true.", 3) - cap.write("end if", 2) - cap.blank_line() - cap.write(f"end {substmt}", 1) - # Write copy_in routine - substmt = f"subroutine {copy_in_funcname}" - cap.blank_line() - cap.write(f"{substmt}(const_array, {err_dummy_str})", 1) - cap.comment("Copy constituent field info into ", 2) - cap.blank_line() - cap.comment("Dummy arguments", 2) - cap.write("real(kind_phys), intent(out) :: const_array(:,:,:)", 2) - for evar in err_vars: - evar.write_def(cap, 2, host, dummy=True, add_intent="out") - # end for - cap.blank_line() - cap.write(f"call {const_obj_name}%copy_in(const_array, {obj_err_callstr})", 2) - cap.write(f"end {substmt}", 1) - # Write copy_out routine - substmt = f"subroutine {copy_out_funcname}" - cap.blank_line() - cap.write(f"{substmt}(const_array, {err_dummy_str})", 1) - cap.comment("Update constituent field info from ", 2) - cap.blank_line() - cap.comment("Dummy arguments", 2) - cap.write("real(kind_phys), intent(in) :: const_array(:,:,:)", 2) - for evar in err_vars: - evar.write_def(cap, 2, host, dummy=True, add_intent="out") - # end for - cap.blank_line() - cap.write(f"call {const_obj_name}%copy_out(const_array, {obj_err_callstr})", 2) - cap.write(f"end {substmt}", 1) - # Write cleanup routine - substmt = f"subroutine {cleanup_funcname}" - cap.blank_line() - cap.write(f"{substmt}()", 1) - cap.comment("Deallocate dynamic constituent array", 2) - cap.blank_line() - for dyn_const in dyn_const_names: - cap.write(f"if (allocated({dyn_const})) then", 2) - cap.write(f"deallocate({dyn_const})", 3) - cap.write("end if", 2) - cap.write(f"call {const_obj_name}%reset()", 2) - # end if - cap.write(f"end {substmt}", 1) - # Write constituents routine - cap.blank_line() - cap.write(f"function {const_array_func}() result(const_ptr)", 1) - cap.blank_line() - cap.comment("Return pointer to constituent array", 2) - cap.blank_line() - cap.comment("Dummy argument", 2) - cap.write("real(kind_phys), pointer :: const_ptr(:,:,:)", 2) - cap.blank_line() - cap.write(f"const_ptr => {const_obj_name}%field_data_ptr()", 2) - cap.write(f"end function {const_array_func}", 1) - # Write advected constituents routine - cap.blank_line() - cap.write(f"function {advect_array_func}() result(const_ptr)", 1) - cap.blank_line() - cap.comment("Return pointer to advected constituent array", 2) - cap.blank_line() - cap.comment("Dummy argument", 2) - cap.write("real(kind_phys), pointer :: const_ptr(:,:,:)", 2) - cap.blank_line() - cap.write(f"const_ptr => {const_obj_name}%advected_constituents_ptr()", - 2) - cap.write(f"end function {advect_array_func}", 1) - # Write the constituent property array routine - cap.blank_line() - cap.write(f"function {prop_array_func}() result(const_ptr)", 1) - cap.write(f"use {CONST_DDT_MOD}, only: {CONST_PROP_PTR_TYPE}", 2) - cap.blank_line() - cap.comment("Return pointer to array of constituent properties", 2) - cap.blank_line() - cap.comment("Dummy argument", 2) - cap.write("type(ccpp_constituent_prop_ptr_t), pointer :: const_ptr(:)", - 2) - cap.blank_line() - cap.write(f"const_ptr => {const_obj_name}%constituent_props_ptr()", - 2) - cap.write(f"end function {prop_array_func}", 1) - # Write constituent index function - substmt = f"subroutine {const_index_func}" - cap.blank_line() - cap.write(f"{substmt}(stdname, const_index, {err_dummy_str})", 1) - cap.comment("Set to the constituent array index " + \ - "for .", 2) - cap.comment("If is not found, set to -1 " + \ - "set an error condition", 2) - cap.blank_line() - cap.comment("Dummy arguments", 2) - cap.write("character(len=*), intent(in) :: stdname", 2) - cap.write("integer, intent(out) :: const_index", 2) - for evar in err_vars: - evar.write_def(cap, 2, host, dummy=True, - add_intent="out", extra_space=1) - # end for - cap.blank_line() - cap.write(f"call {const_obj_name}%const_index(const_index, " + \ - f"stdname, {obj_err_callstr})", 2) - cap.write(f"end {substmt}", 1) - - @staticmethod - def constitutent_source_type(): - """Return the source type for constituent species""" - return ConstituentVarDict.__constituent_type - - @staticmethod - def constituent_prop_array_name(): - """Return the name of the constituent properties array for this suite""" - return ConstituentVarDict.__const_prop_array_name - - @staticmethod - def constituent_prop_init_name(): - """Return the name of the array initialized flag for this suite""" - return ConstituentVarDict.__const_prop_init_name - - @staticmethod - def constituent_prop_init_consts(): - """Return the name of the routine to initialize the constituent - properties array for this suite""" - return ConstituentVarDict.__const_prop_init_consts - - @staticmethod - def write_suite_use(outfile, indent): - """Write use statements for any modules needed by the suite cap. - The statements are written to at indent, . - """ - omsg = f"use ccpp_constituent_prop_mod, only: {CONST_PROP_TYPE}" - outfile.write(omsg, indent) - - @staticmethod - def TF_string(tf_val): - """Return a string of the Fortran equivalent of """ - if tf_val: - tf_str = ".true." - else: - tf_str = ".false." - # end if - return tf_str diff --git a/scripts/conversion_tools/__init__.py b/scripts/conversion_tools/__init__.py deleted file mode 100644 index 4842889a..00000000 --- a/scripts/conversion_tools/__init__.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Public API for the conversion_tools library -""" - -__all__ = [ - 'cm__to__m', - 'm__to__cm', - 'mm__to__m', - 'm__to__mm', - 'um__to__m', - 'm__to__um', - 'm__to__km', - 'km__to__m', - 'mm__to__km', - 'km__to__mm', - 's__to__min', - 'min__to__s', - 's__to__h', - 'h__to__s', - 'h__to__d', - 'd__to__h', - 's__to__d', - 'd__to__s', - 'Pa__to__hPa', - 'hPa__to__Pa', - 'm_s_minus_1__to__km_h_minus_1', - 'km_h_minus_1__to__m_s_minus_1', - 'W_m_minus_2__to__erg_cm_minus_2_s_minus_1', - 'erg_cm_minus_2_s_minus_1__to__W_m_minus_2', - ] - -from .unit_conversion import cm__to__m -from .unit_conversion import m__to__cm -from .unit_conversion import mm__to__m -from .unit_conversion import m__to__mm -from .unit_conversion import um__to__m -from .unit_conversion import m__to__um -from .unit_conversion import m__to__km -from .unit_conversion import km__to__m -from .unit_conversion import mm__to__km -from .unit_conversion import km__to__mm -from .unit_conversion import s__to__min -from .unit_conversion import min__to__s -from .unit_conversion import s__to__h -from .unit_conversion import h__to__s -from .unit_conversion import h__to__d -from .unit_conversion import d__to__h -from .unit_conversion import s__to__d -from .unit_conversion import d__to__s -from .unit_conversion import Pa__to__hPa -from .unit_conversion import hPa__to__Pa -from .unit_conversion import m_s_minus_1__to__km_h_minus_1 -from .unit_conversion import km_h_minus_1__to__m_s_minus_1 -from .unit_conversion import W_m_minus_2__to__erg_cm_minus_2_s_minus_1 -from .unit_conversion import erg_cm_minus_2_s_minus_1__to__W_m_minus_2 diff --git a/scripts/ddt_library.py b/scripts/ddt_library.py deleted file mode 100644 index 1c362108..00000000 --- a/scripts/ddt_library.py +++ /dev/null @@ -1,358 +0,0 @@ -#!/usr/bin/env python3 -# -# Class -# - -"""Module to implement DDT support in the CCPP Framework. -VarDDT is a class to hold all information on a CCPP DDT metadata variable -""" - -# Python library imports -import logging -# CCPP framework imports -from parse_tools import ParseInternalError, CCPPError, context_string -from metavar import Var -from metadata_table import MetadataSection - -############################################################################### - -class VarDDT(Var): - """A class to store a variable that is a component of a DDT (at any - DDT nesting level). - """ - - def __init__(self, new_field, var_ref, run_env, recur=False): - """Initialize a new VarDDT object. - is the DDT component. - is a Var or VarDDT whose root originates in a model - dictionary. - is the CCPPFrameworkEnv object for this framework run. - The structure of the VarDDT object is: - The super class Var object is a copy of the model root Var. - The (a Var) ends up at the end of a VarDDT chain. - """ - self.__field = None - # Grab the info from the root of - source = var_ref.source - super().__init__(var_ref, source, run_env, context=source.context) - # Find the correct place for - if isinstance(var_ref, Var): - # We are at a top level DDT var, set our field - self.__field = new_field - else: - # Recurse to find correct (tail) location for - self.__field = VarDDT(new_field, var_ref.field, run_env, recur=True) - # end if - if ((not recur) and - run_env.verbose): - run_env.logger.debug('Adding DDT field, {}'.format(self)) - # end if - - def is_ddt(self): - """Return True iff is a DDT type.""" - return True - - def get_parent_prop(self, name): - """Return the Var property value for the parent Var object. - """ - return super().get_prop_value(name) - - def get_prop_value(self, name): - """Return the Var property value for the leaf Var object. - """ - if self.field is None: - pvalue = super().get_prop_value(name) - else: - pvalue = self.field.get_prop_value(name) - # end if - return pvalue - - def intrinsic_elements(self, check_dict=None): - """Return the Var intrinsic elements for the leaf Var object. - See Var.intrinsic_elements for details - """ - if self.field is None: - pvalue = super().intrinsic_elements(check_dict=check_dict) - else: - pvalue = self.field.intrinsic_elements(check_dict=check_dict) - # end if - return pvalue - - def clone(self, subst_dict, source_name=None, source_type=None, - context=None): - """Create a clone of this VarDDT object's leaf Var with properties - from overriding this variable's properties. - may also be a string in which case only the local_name - property is changed (to the value of the string). - The optional , , and inputs - allow the clone to appear to be coming from a designated source, - by default, the source and type are the same as this Var (self). - """ - if self.field is None: - clone_var = super().clone(subst_dict, source_name=source_name, - source_type=source_type, context=context) - else: - clone_var = self.field.clone(subst_dict, - source_name=source_name, - source_type=source_type, - context=context) - # end if - return clone_var - - def call_string(self, var_dict, loop_vars=None): - """Return a legal call string of this VarDDT's local name sequence. - """ - # XXgoldyXX: Need to add dimensions to this - call_str = super().get_prop_value('local_name') - if self.field is not None: - call_str += '%' + self.field.call_string(var_dict, - loop_vars=loop_vars) - # end if - return call_str - - def write_def(self, outfile, indent, ddict, allocatable=False, target=False, - dummy=False, add_intent=None, extra_space=0, public=False): - """Write the definition line for this DDT. - The type of this declaration is the type of the Var at the - end of the chain of references.""" - if self.field is None: - super().write_def(outfile, indent, ddict, - allocatable=allocatable, target=target, dummy=dummy, - add_intent=add_intent, extra_space=extra_space, - public=public) - else: - self.field.write_def(outfile, indent, ddict, - allocatable=allocatable, target=target, - dummy=dummy, add_intent=add_intent, - extra_space=extra_space, public=public) - # end if - - @staticmethod - def __var_rep(var, prefix=""): - """Internal helper function for creating VarDDT representations - Create a call string from the local_name and dimensions of . - Optionally, prepend %. - """ - lname = var.get_prop_value('local_name') - ldims = var.get_prop_value('dimensions') - if ldims: - if prefix: - lstr = '{}%{}({})'.format(prefix, lname, ', '.join(ldims)) - else: - lstr = '{}({})'.format(lname, ', '.join(ldims)) - # end if - else: - if prefix: - lstr = '{}%{}'.format(prefix, lname) - else: - lstr = '{}'.format(lname) - # end if - # end if - return lstr - - def __repr__(self): - """Print representation for VarDDT objects""" - # Note, recursion would be messy because of formatting issues - lstr = "" - sep = "" - field = self - while field is not None: - if isinstance(field, VarDDT): - lstr += sep + self.__var_rep(field.var) - field = field.field - elif isinstance(field, Var): - lstr = self.__var_rep(field, prefix=lstr) - field = None - # end if - sep = '%' - # end while - return "".format(lstr) - - def __str__(self): - """Print string for VarDDT objects""" - return self.__repr__() - - @property - def var(self): - "Return this VarDDT's Var object" - return super() - - @property - def field(self): - "Return this objects field object, or None" - return self.__field - -############################################################################### -class DDTLibrary(dict): - """DDTLibrary is a collection of DDT definitions, broken down into - individual fields with metadata. It provides efficient ways to find - the field corresponding to any standard-named field contained in - any of the (potentially nested) included DDT definitions. - The dictionary holds known standard names. - """ - - def __init__(self, name, run_env, ddts=None): - "Our dict is DDT definition headers, key is type" - self.__name = '{}_ddt_lib'.format(name) -# XXgoldyXX: v remove? -# self.__ddt_fields = {} # DDT field to DDT access map -# XXgoldyXX: ^ remove? - self.__max_mod_name_len = 0 - self.__run_env = run_env - super().__init__() - if ddts is None: - ddts = list() - elif not isinstance(ddts, list): - ddts = [ddts] - # end if - # Add all the DDT headers, then process - for ddt in ddts: - if not isinstance(ddt, MetadataSection): - errmsg = 'Invalid DDT metadata type, {}' - raise ParseInternalError(errmsg.format(type(ddt).__name__)) - # end if - if not ddt.header_type == 'ddt': - errmsg = 'Metadata table header is for a {}, should be DDT' - raise ParseInternalError(errmsg.format(ddt.header_type)) - # end if - if ddt.title in self: - errmsg = "Duplicate DDT, {}, found{}, original{}" - ctx = context_string(ddt.context) - octx = context_string(self[ddt.title].context) - raise CCPPError(errmsg.format(ddt.title, ctx, octx)) - # end if - if run_env.verbose: - lmsg = f"Adding DDT {ddt.title} to {self.name}" - run_env.logger.debug(lmsg) - # end if - self[ddt.title] = ddt - dlen = len(ddt.module) - if dlen > self.__max_mod_name_len: - self.__max_mod_name_len = dlen - # end if - # end for - - def check_ddt_type(self, var, header, lname=None): - """If is a DDT, check to make sure it is in this DDT library. - If not, raise an exception. - """ - if var.is_ddt(): - # Make sure we know this DDT type - vtype = var.get_prop_value('type') - if vtype not in self: - if lname is None: - lname = var.get_prop_value('local_name') - # end if - errmsg = 'Variable {} is of unknown type ({}) in {}' - ctx = context_string(var.context) - raise CCPPError(errmsg.format(lname, vtype, header.title, ctx)) - # end if - # end if (no else needed) - - def collect_ddt_fields(self, var_dict, var, run_env, - ddt=None, skip_duplicates=False, parent=None): - """Add all the reachable fields from DDT variable of type, - to . Each field is added as a VarDDT. - If , add VarDDT recursively using parent. - Note: By default, it is an error to try to add a duplicate - field to (i.e., the field already exists in - or one of its parents). To simply skip duplicate - fields, set to True. - """ - if ddt is None: - vtype = var.get_prop_value('type') - if vtype in self: - ddt = self[vtype] - else: - lname = var.get_prop_value('local_name') - ctx = context_string(var.context) - errmsg = "Variable, {}, is not a known DDT{}" - raise ParseInternalError(errmsg.format(lname, ctx)) - # end if - # end if - for dvar in ddt.variable_list(): - if parent is None: - subvar = VarDDT(dvar, var, self.run_env) - else: - subvar = VarDDT(VarDDT(dvar, var, self.run_env), parent, self.run_env) - # end if - dvtype = dvar.get_prop_value('type') - if (dvar.is_ddt()) and (dvtype in self): - # If DDT in our library, we need to add sub-fields recursively. - subddt = self[dvtype] - self.collect_ddt_fields(var_dict, dvar, run_env, parent=var, ddt=subddt) - # end if - # add_variable only checks the current dictionary. By default, - # for a DDT, the variable also cannot be in our parent - # dictionaries. - stdname = dvar.get_prop_value('standard_name') - pvar = var_dict.find_variable(standard_name=stdname, any_scope=True) - if pvar and (not skip_duplicates): - ntx = context_string(dvar.context) - ctx = context_string(pvar.context) - emsg = f"Attempt to add duplicate DDT sub-variable, {stdname}{ntx}." - emsg += f"\nVariable originally defined{ctx}" - raise CCPPError(emsg.format(stdname, ntx, ctx)) - # end if - # Add this intrinsic to - if not pvar: - var_dict.add_variable(subvar, run_env) - # end if - # end for - - def ddt_modules(self, variable_list, ddt_mods=None): - """Collect information for module use statements. - Add module use information (module name, DDT name) for any variable - in which is a DDT in this library. - """ - if ddt_mods is None: - ddt_mods = set() # Need a new set for every call - # end if - for var in variable_list: - vtype = var.get_prop_value('type') - if vtype in self: - module = self[vtype].module - ddt_mods.add((module, vtype)) - # end if - # end for - return ddt_mods - - def write_ddt_use_statements(self, variable_list, outfile, indent, pad=0): - """Write the use statements for all ddt modules needed by - """ - pad = max(pad, self.__max_mod_name_len) - ddt_mods = self.ddt_modules(variable_list) - for ddt_mod in ddt_mods: - dmod = ddt_mod[0] - dtype = ddt_mod[1] - slen = ' '*(pad - len(dmod)) - ustring = 'use {},{} only: {}' - outfile.write(ustring.format(dmod, slen, dtype), indent) - # end for - - @property - def name(self): - """Return the name of this DDT library""" - return self.__name - - @property - def run_env(self): - """Return the CCPPFrameworkEnv object for this DDT library""" - return self.__run_env - - @property - def max_mod_name_len(self): - """Return the maximum module name length of this DDT library's modules""" - return self.__max_mod_name_len - -############################################################################### -if __name__ == "__main__": - # pylint: disable=ungrouped-imports - import doctest - import sys - # pylint: enable=ungrouped-imports - fail, _ = doctest.testmod() - sys.exit(fail) -# end if diff --git a/scripts/file_utils.py b/scripts/file_utils.py deleted file mode 100644 index 1342f72b..00000000 --- a/scripts/file_utils.py +++ /dev/null @@ -1,303 +0,0 @@ -#!/usr/bin/env python3 - -""" -Utilities for checking and manipulating file status -""" - -# Python library imports -import filecmp -import glob -import os -# CCPP framework imports -from parse_tools import CCPPError, ParseInternalError - -# Standardize name of generated kinds file and module -KINDS_MODULE = 'ccpp_kinds' -KINDS_FILENAME = '{}.F90'.format(KINDS_MODULE) - -############################################################################### -def check_for_existing_file(filename, description, readable=True): -############################################################################### - """Check for file existence and access. - Return a list of error strings in case - does not exist or does not have read access and is True""" - errors = list() - if os.path.exists(filename): - if readable: - if not os.access(filename, os.R_OK): - errmsg = "No read access to {}, '{}'" - errors.append(errmsg.format(description, filename)) - # end if (no else, everything is fine) - # end if (no else, everything is fine) - else: - errors.append("{}, '{}', must exist".format(description, filename)) - # end if - return errors - -############################################################################### -def check_for_writeable_file(filename, description): -############################################################################### - """If exists but not writable, raise an error. - If does not exist and its directory is not writable, raise - an error. is a description of .""" - if os.path.exists(filename) and not os.access(filename, os.W_OK): - raise CCPPError("Cannot write {}, '{}'".format(description, filename)) - # end if - if not os.access(os.path.dirname(filename), os.W_OK): - raise CCPPError("Cannot write {}, '{}'".format(description, filename)) - # end if (else just return) - -############################################################################### -def add_unique_files(filepath, pdesc, master_list, logger): -############################################################################### - """Add any new files indicated by to . - Check each file for readability. - Log duplicate files - Return a list of errors found - Wildcards in are expanded""" - errors = list() - for file in glob.glob(filepath): - errs = check_for_existing_file(file, pdesc) - if errs: - errors.extend(errs) - elif file in master_list: - lmsg = "WARNING: Ignoring duplicate file, {}" - logger.warning(lmsg.format(file)) - else: - master_list.append(file) - # end if - # end for - return errors - -############################################################################### -def read_pathnames_from_file(pathsfile, file_type): -############################################################################### - """Read and return path names from . - Convert relative pathnames to use 's directory as root. - Also return a list of any errors encountered - """ - # We want to end up with absolute paths, treat as root location - root_path = os.path.dirname(os.path.abspath(pathsfile)) - file_list = list() - pdesc = '{} pathsnames file'.format(file_type) - errors = check_for_existing_file(pathsfile, pdesc) - pdesc = '{} pathname in {}'.format(file_type, pathsfile) - if not errors: - with open(pathsfile, 'r') as infile: - for line in infile.readlines(): - path = line.strip() - # Skip blank lines & lines which appear to start with a comment. - if path and (path[0] not in ['#', '!']): - # Check for an absolute path - if not os.path.isabs(path): - path = os.path.normpath(os.path.join(root_path, path)) - # end if - file_list.append(path) - # end if (else skip blank or comment line) - # end for - # end with open - # end if (no else, we already have the errors) - return file_list, errors - -############################################################################### -def _create_file_list_int(files, suffices, file_type, logger, - txt_files, pathname, root_path, master_list): -############################################################################### - """Create and return a master list of files from . - is a list of pathnames which may include wildcards. - is a list of allowed file types. Filenames in - with an allowed suffix will be added to the master list. - Filenames with a '.txt' suffix will be parsed to look for allowed - filenames. - is a description of the allowed file types. - is a logger used to print warnings (unrecognized filename types) - and debug messages. - is a list of previously-encountered text files (to prevent - infinite recursion). - is the text file name from which was read (if any). - is the list of files which have already been collected - A list of error strings is also returned - """ - errors = list() - if pathname: - pdesc = '{} pathname file, found in {}'.format(file_type, pathname) - else: - pdesc = '{} pathnames file'.format(file_type) - # end if - if not isinstance(files, list): - raise ParseInternalError("'{}' is not a list".format(files)) - # end if - for filename in files: - # suff is filename's extension - suff = os.path.splitext(filename)[1] - if suff: - suff = suff[1:] - # end if - if not os.path.isabs(filename): - filename = os.path.normpath(os.path.join(root_path, filename)) - # end if - if os.path.isdir(filename): - for suff_type in suffices: - file_type = os.path.join(filename, '*.{}'.format(suff_type)) - errs = add_unique_files(file_type, pdesc, master_list, logger) - errors.extend(errs) - # end for - elif suff in suffices: - errs = add_unique_files(filename, pdesc, master_list, logger) - errors.extend(errs) - elif suff == 'txt': - tfiles = glob.glob(filename) - if tfiles: - for file in tfiles: - if file in txt_files: - lmsg = "WARNING: Ignoring duplicate '.txt' file, {}" - logger.warning(lmsg.format(filename)) - else: - lmsg = 'Reading .{} filenames from {}' - logger.debug(lmsg.format(', .'.join(suffices), - file)) - flist, errs = read_pathnames_from_file(file, file_type) - errors.extend(errs) - txt_files.append(file) - root = os.path.dirname(file) - _, errs = _create_file_list_int(flist, suffices, - file_type, logger, - txt_files, file, - root, master_list) - errors.extend(errs) - # end if - # end for - else: - emsg = "{} pathnames file, '{}', does not exist" - errors.append(emsg.format(file_type, filename)) - # end if - else: - lmsg = 'WARNING: Not reading {}, only reading .{} or .txt files' - logger.warning(lmsg.format(filename, ', .'.join(suffices))) - # end if - # end for - - return master_list, errors - -############################################################################### -def create_file_list(files, suffices, file_type, logger, root_path=None): -############################################################################### - """Create and return a master list of files from . - is either a comma-separated string of pathnames or a list. - If a pathname is a directory, all files with extensions in - are included. - Wildcards in a pathname are expanded. - is a list of allowed file types. Filenames in - with an allowed suffix will be added to the master list. - Filenames with a '.txt' suffix will be parsed to look for allowed - filenames. - is a description of the allowed file types. - is a logger used to print warnings (unrecognized filename types) - and debug messages. - If is not None, it is used to create absolute paths for - , otherwise, the current working directory is used. - """ - master_list = list() - txt_files = list() # Already processed txt files - pathname = None - if isinstance(files, str): - file_list = [x.strip() for x in files.split(',') if x.strip()] - elif isinstance(files, (list, tuple)): - file_list = files - else: - raise ParseInternalError("Bad input, = {}".format(files)) - # end if - if root_path is None: - root_path = os.getcwd() - # end if - master_list, errors = _create_file_list_int(file_list, suffices, file_type, - logger, txt_files, pathname, - root_path, master_list) - if errors: - emsg = 'Error processing list of {} files:\n {}' - raise CCPPError(emsg.format(file_type, '\n '.join(errors))) - # end if - return master_list - -############################################################################### -def replace_paths(dir_list, src_dir, dest_dir): -############################################################################### - """For every path in , replace instances of with - """ - for index, path in enumerate(dir_list): - dir_list[index] = path.replace(src_dir, dest_dir) - # end for - -############################################################################### -def remove_dir(src_dir, force=False): -############################################################################### - """Remove and its children. This operation can only succeed if - contains no files or if is True.""" - currdir = os.getcwd() - src_parent = os.path.split(src_dir)[0] - src_rel = os.path.relpath(src_dir, src_parent) - os.chdir(src_parent) # Prevent removing the parent of src_dir - if force: - leaf_dirs = set() - for root, dirs, files in os.walk(src_rel): - for file in files: - os.remove(os.path.join(root, file)) - # end for - if not dirs: - leaf_dirs.add(root) - # end if - # end for - for ldir in leaf_dirs: - os.removedirs(ldir) - # end for - # end if (no else, always try to remove top level - try: - os.removedirs(src_rel) - except OSError: - pass # Ignore error, fail silently - # end try - os.chdir(currdir) - -############################################################################### -def move_modified_files(src_dir, dest_dir, overwrite=False, remove_src=False): -############################################################################### - """For each file in , move it to if that file is - different in the two locations. - if is True, move all files to , even if unchanged. - If is True, remove when complete.""" - src_files = {} # All files in - if os.path.normpath(src_dir) != os.path.normpath(dest_dir): - for root, _, files in os.walk(src_dir): - for file in files: - src_path = os.path.join(root, file) - if file in src_files: - # We do not allow two files with the same name - emsg = "Duplicate CCPP file found, '{}', original is '{}'" - raise CCPPError(emsg.format(src_path, src_files[file])) - # end if - src_files[file] = src_path - # end for - # end for - for file in src_files: - src_path = src_files[file] - src_file = os.path.relpath(src_path, start=src_dir) - dest_path = os.path.join(dest_dir, src_file) - if os.path.exists(dest_path): - if overwrite: - fmove = True - else: - fmove = not filecmp.cmp(src_path, dest_path, shallow=False) - # end if - else: - fmove = True - # end if - if fmove: - os.replace(src_path, dest_path) - else: - os.remove(src_path) - # end if - # end for - if remove_src: - remove_dir(src_dir, force=True) - # end if - # end if (no else, take no action if the directories are identical) diff --git a/scripts/fortran_tools/__init__.py b/scripts/fortran_tools/__init__.py deleted file mode 100644 index 8626a9b1..00000000 --- a/scripts/fortran_tools/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Public API for the fortran_parser library -""" -import sys -import os.path -sys.path.insert(0, os.path.dirname(__file__)) - -# pylint: disable=wrong-import-position -from parse_fortran_file import parse_fortran_file -from parse_fortran import parse_fortran_var_decl, fortran_type_definition -from fortran_write import FortranWriter -# pylint: enable=wrong-import-position - -__all__ = [ - 'fortran_type_definition', - 'parse_fortran_file', - 'parse_fortran_var_decl', - 'FortranWriter' -] diff --git a/scripts/fortran_tools/fortran_write.py b/scripts/fortran_tools/fortran_write.py deleted file mode 100644 index 35d403e0..00000000 --- a/scripts/fortran_tools/fortran_write.py +++ /dev/null @@ -1,436 +0,0 @@ -#!/usr/bin/env python3 -# - -"""Code to write Fortran code -""" - -import math - -class FortranWriter: - """Class to turn output into properly continued and indented Fortran code - >>> FortranWriter("foo.F90", 'r', 'test', 'mod_name') #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ValueError: Read mode not allowed in FortranWriter object - >>> FortranWriter("foo.F90", 'wb', 'test', 'mod_name') #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ValueError: Binary mode not allowed in FortranWriter object - """ - - ########################################################################### - # Class variables - ########################################################################### - __INDENT = 2 # Spaces per indent level - - __CONTINUE_INDENT = 4 # Extra spaces on continuation line - - __LINE_FILL = 97 # Target line length - - __LINE_MAX = 120 # Max line length (for Codee) - - __BREAK_CHARS = [',', '+', '*', '/', '(', ')'] - - # CCPP copyright statement to be included in all generated Fortran files - __COPYRIGHT = '''! -! This work (Common Community Physics Package Framework), identified by -! NOAA, NCAR, CU/CIRES, is free of known copyright restrictions and is -! placed in the public domain. -! -! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -! THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.''' - - __MOD_HEADER = ''' -!> -!! @brief Auto-generated {file_desc} -!! -! -module {module} -''' - - __MOD_PREAMBLE = ["implicit none", "private"] - - __CONTAINS = ''' -contains''' - - __MOD_FOOTER = ''' -end module {module}''' - - ########################################################################### - - def indent(self, level=0, continue_line=False): - 'Return an indent string for any level' - indent = self.indent_size * level - if continue_line: - indent = indent + self.__continue_indent - # End if - return indent*' ' - - ########################################################################### - - def find_best_break(self, choices, last=None): - """Find the best line break point given . - If is present, use it as a target line length.""" - if last is None: - last = self.__line_fill - # End if - # Find largest good break - possible = [x for x in choices if 0 < x < last] - if not possible: - best = self.__line_max + 1 - else: - best = max(possible) - # End if - if (best > self.__line_max) and (last < self.__line_max): - best = self.find_best_break(choices, last=self.__line_max) - # End if - return best - - ########################################################################### - - @staticmethod - def _in_quote(test_str): - """Return True if ends in a character context. - >>> FortranWriter._in_quote("hi'mom") - True - >>> FortranWriter._in_quote("hi mom") - False - >>> FortranWriter._in_quote("'hi mom'") - False - >>> FortranWriter._in_quote("'hi"" mom'") - False - """ - in_single_char = False - in_double_char = False - for char in test_str: - if in_single_char: - if char == "'": - in_single_char = False - # end if - elif in_double_char: - if char == '"': - in_double_char = False - # end if - elif char == "'": - in_single_char = True - elif char == '"': - in_double_char = True - # end if - # end for - return in_single_char or in_double_char - - ########################################################################### - - def write(self, statement, indent_level, continue_line=False): - """Write to the open file, indenting to - (see self.indent). - If is True, treat this line as a continuation of - a previous statement.""" - if isinstance(statement, list): - for stmt in statement: - self.write(stmt, indent_level, continue_line) - # End for - elif '\n' in statement: - for stmt in statement.split('\n'): - self.write(stmt, indent_level, continue_line) - # End for - else: - istr = self.indent(indent_level, continue_line) - ostmt = statement.strip() - is_comment_stmt = ostmt and (ostmt[0] == '!') - in_comment = "" - outstr = istr + ostmt - line_len = len(outstr) - if line_len > self.__line_fill: - # Collect pretty break points - spaces = [] - break_chars = [] - sptr = len(istr) - in_single_char = False - in_double_char = False - while sptr < line_len: - if in_single_char: - if outstr[sptr] == "'": - in_single_char = False - # End if (no else, just copy stuff in string) - elif in_double_char: - if outstr[sptr] == '"': - in_double_char = False - # End if (no else, just copy stuff in string) - elif outstr[sptr] == "'": - in_single_char = True - elif outstr[sptr] == '"': - in_double_char = True - elif outstr[sptr] == '!': - # Comment in non-character context - spaces.append(sptr-1) - in_comment = "! " # No continue for comment - if ((not is_comment_stmt) and - (sptr >= self.__max_comment_start)): - # suck in rest of line - sptr = line_len - 1 - # end if - elif outstr[sptr] == ' ': - # Non-quote spaces are where we can break - spaces.append(sptr) - elif outstr[sptr:sptr+2] == '//': - # Non-quote syntax are where we can break - break_chars.append(sptr + 1) - elif outstr[sptr] in FortranWriter.__BREAK_CHARS: - # Non-quote syntax are where we can break - break_chars.append(sptr) - # End if (no else, other characters will be ignored) - sptr = sptr + 1 - # End while - # Before looking for best space, reject any that are on a - # comment line but before any significant characters - if outstr.lstrip().startswith('!'): - first_space = outstr.index('!') + 1 - while ((outstr[first_space] == '!' or - outstr[first_space] == ' ') and - (first_space < line_len)): - first_space += 1 - # end while - if min(spaces) < first_space: - spaces = [x for x in spaces if x >= first_space] - # end if - best = self.find_best_break(spaces) - if best >= self.__line_fill: - best = min(best, self.find_best_break(break_chars)) - # End if - line_continue = False - if best >= self.__line_max: - # This is probably a bad situation so we have to break - # in an ugly spot - best = self.__line_max - 1 - if len(outstr) > best: - line_continue = '&' - # end if - # end if - if len(outstr) > best: - if self._in_quote(outstr[0:best+1]): - if best >= FortranWriter.__LINE_MAX - 1: - best = FortranWriter.__LINE_MAX - 2 - # end if - line_continue = '&' - elif not outstr[best+1:].lstrip(): - # If the next line is empty, the current line is done - # and is equal to the max line length. Do not use - # continue and set best to line_max (best+1) - line_continue = False - best = best+1 - else: - # If next line is just comment, do not use continue - line_continue = outstr[best+1:].lstrip()[0] != '!' - # end if - elif not line_continue: - line_continue = len(outstr) > best - # End if - if in_comment or is_comment_stmt: - line_continue = False - # end if - if line_continue == '&': - fill = '&' - elif line_continue: - fill = ' &' - else: - fill = "" - # End if - outline = f"{outstr[0:best+1].rstrip()}{fill}" - self.__file.write(f"{outline}\n") - if best <= 0: - imsg = "Internal ERROR: Unable to break line" - raise ValueError(f"{imsg}, '{statement}'") - # end if - statement = in_comment + outstr[best+1:] - if isinstance(line_continue, str) and statement.strip(): - statement = line_continue + statement - # end if - if statement.strip(): - self.write(statement, indent_level, continue_line=line_continue) - else: - self.__file.write(f"{outstr}\n") - # End if - # End if - - ########################################################################### - - def __init__(self, filename, mode, file_description, module_name, - indent=None, continue_indent=None, - line_fill=None, line_max=None): - """Initialize thie FortranWriter object. - Some boilerplate is written automatically.""" - self.__file_desc = file_description.replace('\n', '\n!! ') - self.__module = module_name - # We only handle writing situations (for now) and only text - if 'r' in mode: - raise ValueError('Read mode not allowed in FortranWriter object') - # end if - if 'b' in mode: - raise ValueError('Binary mode not allowed in FortranWriter object') - # End if - self.__file = open(filename, mode) - if indent is None: - self.__indent = FortranWriter.__INDENT - else: - self.__indent = indent - # End if - if continue_indent is None: - self.__continue_indent = FortranWriter.__CONTINUE_INDENT - else: - self.__continue_indent = continue_indent - # End if - if line_fill is None: - self.__line_fill = FortranWriter.__LINE_FILL - else: - self.__line_fill = line_fill - # End if - self.__max_comment_start = math.ceil(self.__line_fill * 3 / 4) - if line_max is None: - self.__line_max = FortranWriter.__LINE_MAX - else: - self.__line_max = line_max - # End if - - ########################################################################### - - def write_preamble(self): - """Write the module boilerplate that goes between use statements - and module declarations.""" - self.write("", 0) - for stmt in FortranWriter.__MOD_PREAMBLE: - self.write(stmt, 1) - # end for - self.write("", 0) - - ########################################################################### - - def end_module_header(self): - """Write the module contains statement.""" - self.write(FortranWriter.__CONTAINS, 0) - - ########################################################################### - - def __enter__(self, *args): - self.write(FortranWriter.__COPYRIGHT, 0) - self.write(self.module_header(), 0) - return self - - ########################################################################### - - def __exit__(self, *args): - self.write(FortranWriter.__MOD_FOOTER.format(module=self.__module), 0) - self.__file.close() - return False - - ########################################################################### - - def module_header(self): - """Return the standard Fortran module header for and - """ - return FortranWriter.__MOD_HEADER.format(file_desc=self.__file_desc, - module=self.__module) - - ########################################################################### - - def comment(self, comment, indent): - """Write a Fortran comment with contents, """ - mlcomment = comment.replace('\n', '\n! ') # No backslash in f string - self.write(f"! {mlcomment}", indent) - - ########################################################################### - - def blank_line(self): - """Write a blank line""" - self.write("", 0) - - ########################################################################### - - def include(self, filename): - """Insert the contents of verbatim.""" - with open(filename, 'r') as infile: - for line in infile: - self.__file.write(line) - # end for - # end with - - ########################################################################### - - @property - def line_fill(self): - """Return the target line length for this Fortran file""" - return self.__line_fill - - ########################################################################### - - @property - def indent_size(self): - """Return the number of spaces for each indent level for this - Fortran file - """ - return self.__indent - - ########################################################################### - - @classmethod - def copyright(cls): - """Return the standard Fortran file copyright string""" - return cls.__COPYRIGHT - -############################################################################### -if __name__ == "__main__": - # First, run doctest - # pylint: disable=ungrouped-imports - import doctest - import os - import sys - # pylint: enable=ungrouped-imports - fail, _ = doctest.testmod() - # Make sure we can write a file - sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - NAME = 'foo' - while os.path.exists(NAME+'.F90'): - NAME = NAME + 'xo' - # end while - NAME = NAME + '.F90' - if os.access(os.getcwd(), os.W_OK): - _CHECK = FortranWriter.copyright().split('\n') - with FortranWriter(NAME, 'w', 'doctest', 'foo') as foo: - foo.write_preamble() - foo.end_module_header() - foo.write(("subroutine foo(long_argument1, long_argument2, " - "long_argument3, long_argument4, long_argument5)"), 2) - foo.write("end subroutine foo", 2) - _CHECK.extend(foo.module_header().rstrip().split('\n')) - # End with - _CHECK.extend(["", "", " implicit none", " private", - "", "", "CONTAINS"]) - _CHECK.extend([(' subroutine foo(long_argument1, long_argument2, ' - 'long_argument3, long_argument4, &'), - ' long_argument5)', - ' end subroutine foo', '', - 'end module foo']) - # Check file - with open(NAME, 'r') as foo: - _STATEMENTS = foo.readlines() - if len(_STATEMENTS) != len(_CHECK): - EMSG = "ERROR: File has {} statements, should have {}" - print(EMSG.format(len(_STATEMENTS), len(_CHECK))) - else: - for _line_num, _statement in enumerate(_STATEMENTS): - if _statement.rstrip() != _CHECK[_line_num]: - EMSG = "ERROR: Line {} does not match" - print(EMSG.format(_line_num+1)) - print("{}".format(_statement.rstrip())) - print("{}".format(_CHECK[_line_num])) - # end if - # end for - # end with - os.remove(NAME) - else: - print("WARNING: Unable to write test file, '{}'".format(NAME)) - # end if - sys.exit(fail) -# end if diff --git a/scripts/fortran_tools/offline_check_fortran_vs_metadata.py b/scripts/fortran_tools/offline_check_fortran_vs_metadata.py deleted file mode 100755 index ef50ced7..00000000 --- a/scripts/fortran_tools/offline_check_fortran_vs_metadata.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 - -""" -Recursively compare all fortran and metadata files in user-supplied directory, and report any problems -USAGE: - ./offline_check_fortran_vs_metadata.py --directory (--debug) -""" - - -import sys -import os -import glob -import logging -import argparse -import site -# Enable imports from parent directory -site.addsitedir(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -# CCPP framework imports -from framework_env import CCPPFrameworkEnv -from fortran_tools import parse_fortran_file -from metadata_table import parse_metadata_file -from ccpp_capgen import find_associated_fortran_file -from ccpp_capgen import parse_scheme_files -from parse_tools import init_log, set_log_level -from parse_tools import register_fortran_ddt_name -from parse_tools import CCPPError, ParseInternalError - -_LOGGER = init_log(os.path.basename(__file__)) -_DUMMY_RUN_ENV = CCPPFrameworkEnv(_LOGGER, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - -def find_files_to_compare(directory): - metadata_files = [] - for file in glob.glob(os.path.join(directory,'**','*.meta'), recursive=True): - metadata_files.append(file) - # end for - return metadata_files - -def compare_fortran_and_metadata(scheme_directory, run_env): - ## Check for files - metadata_files = find_files_to_compare(scheme_directory) - # Perform checks - parse_scheme_files(metadata_files, run_env, skip_ddt_check=True, relative_source_path=True) - -def parse_command_line(arguments, description): - """Parse command-line arguments""" - parser = argparse.ArgumentParser(description=description, - formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument("--directory", type=str, required=True, - metavar='top-level directory to analyze - REQUIRED', - help="""Full path to scheme directory""") - parser.add_argument("--debug", action='store_true', default=False, - help="""turn on debug mode for additional verbosity""") - pargs = parser.parse_args(arguments) - return pargs - -def _main_func(): - """Parse command line, then parse indicated host, scheme, and suite files. - Finally, generate code to allow host model to run indicated CCPP suites.""" - pargs = parse_command_line(sys.argv[1:], __doc__) - logger = _LOGGER - if pargs.debug: - set_log_level(logger, logging.DEBUG) - else: - set_log_level(logger, logging.INFO) - # end if - compare_fortran_and_metadata(pargs.directory, _DUMMY_RUN_ENV) - print('All checks passed!') - -############################################################################### - -if __name__ == "__main__": - try: - _main_func() - sys.exit(0) - except ParseInternalError as pie: - _LOGGER.exception(pie) - sys.exit(-1) - except CCPPError as ccpp_err: - if _LOGGER.getEffectiveLevel() <= logging.DEBUG: - _LOGGER.exception(ccpp_err) - else: - _LOGGER.error(ccpp_err) - # end if - sys.exit(1) - finally: - logging.shutdown() - # end try - diff --git a/scripts/fortran_tools/parse_fortran.py b/scripts/fortran_tools/parse_fortran.py deleted file mode 100644 index 00e22fce..00000000 --- a/scripts/fortran_tools/parse_fortran.py +++ /dev/null @@ -1,914 +0,0 @@ -#!/usr/bin/env python3 - -"""Types and code for parsing Fortran source code. -""" - -# pylint: disable=wrong-import-position -if __name__ == '__main__' and __package__ is None: - import sys - import os.path - sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import re -from parse_tools import ParseSyntaxError, ParseInternalError -from parse_tools import ParseContext, context_string -from parse_tools import check_fortran_intrinsic -from parse_tools import check_balanced_paren, unique_standard_name -#pylint: disable=unused-import -from parse_tools import ParseSource # Used in doctest -#pylint: enable=unused-import -from metavar import FortranVar -# pylint: enable=wrong-import-position - -# A collection of types and tools for parsing Fortran code to support -# CCPP metadata parsing. The purpose of this code is limited to type -# checking of routines with CCPP metadata caps, therefore full routines are -# not parsed and a full Fortran syntax tree is not warranted. - -######################################################################## - -# Fortran ID specifier (do not want a group like FORTRAN_ID from parse_tools) -_FORTRAN_ID = r"(?:[A-Za-z][A-Za-z0-9_]*)" -# Regular expression for a dimension specifier -_DIMID = r"(?:"+_FORTRAN_ID+r"|[0-9]+)" -_DIMCOLON = r"(?:\s*:\s*"+_DIMID+r"?\s*)" -_DIMCOLONS = r"(?:"+_DIMID+r"?"+_DIMCOLON+_DIMCOLON+r"?)" -_DIMSPEC = r"(?:"+_DIMID+r"|"+_DIMCOLONS+r")" -_dims_list_ = _DIMSPEC+r"(?:\s*,\s*"+_DIMSPEC+r"){0,6}" -# Regular expression for a variable name with optional dimensions -_VAR_ID_RE = re.compile(r"("+_FORTRAN_ID+r")\s*(\(\s*"+_dims_list_+r"\s*\))?$") - -######################################################################## - -class Ftype(object): - """Ftype is the base class for all Fortran types - It is also the final type for intrinsic types except for character - >>> Ftype('integer').typestr - 'integer' - >>> Ftype('integer', kind_in='( kind= I8').__str__() #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Invalid kind_selector, '( kind= I8', at :1 - >>> Ftype('integer', kind_in='(kind=I8)').__str__() - 'integer(kind=I8)' - >>> Ftype('integer', kind_in='(I8)').__str__() - 'integer(kind=I8)' - >>> Ftype('real', kind_in='(len=*,R8)').__str__() #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Invalid kind_selector, '(len=*,R8)', at :1 - >>> Ftype(typestr_in='real', line_in='real(kind=kind_phys)') #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: typestr_in and line_in cannot both be used in a single call, at :1 - >>> Ftype(typestr_in='real', line_in='real(kind=kind_phys)', context=ParseContext(linenum=37, filename="foo.F90")) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: typestr_in and line_in cannot both be used in a single call, at foo.F90:37 - >>> Ftype(line_in='real(kind=kind_phys)').__str__() - 'real(kind=kind_phys)' - >>> Ftype(line_in="integer").__str__() - 'integer' - >>> Ftype(line_in="INTEGER").__str__() - 'INTEGER' - """ - - # Note that "character" is not in intrinsic_types even though it is a - # Fortran intrinsic. This is because character has its own type. - __intrinsic_types__ = [r"integer", r"real", r"logical", - r"double\s*precision", r"complex"] - - __itype_re = re.compile(r"(?i)({})\s*(\([A-Za-z0-9,=_\s]+\))?".format(r"|".join(__intrinsic_types__))) - __kind_re = re.compile(r"(?i)kind\s*(\()?\s*([\'\"])?(.+?)([\'\"])?\s*(\))?") - - __attr_spec = ['allocatable', 'asynchronous', 'dimension', 'external', - 'intent', 'intrinsic', 'bind', 'optional', 'parameter', - 'pointer', 'private', 'protected', 'public', 'save', - 'target', 'value', 'volatile'] - - def __init__(self, typestr_in=None, kind_in=None, match_len_in=None, - line_in=None, context=None): - """Initialize this FType object, either using and - , OR using line_in.""" - if context is None: - self.__context = ParseContext() - else: - self.__context = ParseContext(context=context) - # end if - # We have to distinguish which type of initialization we have - self.__typestr = typestr_in - if typestr_in is not None: - if line_in is not None: - emsg = "Cannot pass both typestr_in and line_in as arguments" - raise ParseInternalError(emsg, self.__context) - # end if - self.__default_kind = kind_in is None - if kind_in is None: - self.__kind = None - elif kind_in[0] == '(': - # Parse an explicit kind declaration - self.__kind = self.parse_kind_selector(kind_in) - else: - # The kind has already been parsed for us (e.g., by character) - self.__kind = kind_in - # end if - if match_len_in is not None: - self.__match_len = match_len_in - else: - self.__match_len = len(self.typestr) - if kind_in is not None: - self.__match_len += len(self.__kind) + 2 - # end if - # end if - elif kind_in is not None: - emsg = "kind_in cannot be passed without typestr_in" - raise ParseInternalError(emsg, self.__context) - elif line_in is not None: - match = Ftype.type_match(line_in) - if match is None: - emsg = "type declaration" - raise ParseSyntaxError(emsg, token=line_in, - context=self.__context) - # end if - if match_len_in is not None: - self.__match_len = match_len_in - else: - self.__match_len = len(match.group(0)) - # end if - if check_fortran_intrinsic(match.group(1)): - self.__typestr = match.group(1) - if match.group(2) is not None: - # Parse kind section - kmatch = match.group(2).strip() - self.__kind = self.parse_kind_selector(kmatch) - else: - self.__kind = None - # end if - self.__default_kind = self.__kind is None - else: - raise ParseSyntaxError("type declaration", - token=line_in, context=self.__context) - # end if - else: - emsg = "At least one of typestr_in or line_in must be passed" - raise ParseInternalError(emsg, self.__context) - # end if - - def parse_kind_selector(self, kind_selector, context=None): - """Find and return the 'kind' value from - '(foo)' and '(kind=foo)' both return 'foo'""" - if context is None: - if hasattr(self, 'context'): - context = self.__context - else: - context = ParseContext() - # end if - kind = None - if (kind_selector[0] == '(') and (kind_selector[-1] == ')'): - args = kind_selector[1:-1].split('=') - else: - args = kind_selector.split('=') - # end if - if (len(args) > 2) or (len(args) < 1): - raise ParseSyntaxError("kind_selector", - token=kind_selector, context=context) - # end if - if len(args) == 1: - kind = args[0].strip() - elif args[0].strip().lower() != 'kind': - # We have two args, the first better be kind - raise ParseSyntaxError("kind_selector", - token=kind_selector, context=context) - # end if - if kind is None: - # We have two args and the second is our kind string - kind = args[1].strip() - # end if - # One last check for missing right paren - match = Ftype.__kind_re.search(kind) - if match is not None: - if match.group(2) is not None: - if match.group(2) != match.group(4): - raise ParseSyntaxError("kind_selector", - token=kind_selector, context=context) - # end if - if (match.group(1) is None) and (match.group(5) is not None): - raise ParseSyntaxError("kind_selector", - token=kind_selector, context=context) - # end if - if (match.group(1) is not None) and (match.group(5) is None): - raise ParseSyntaxError("kind_selector", - token=kind_selector, context=context) - # end if - else: - pass - elif kind[0:4].lower() == "kind": - # Got 'something' == 'kind'?? - raise ParseSyntaxError("kind_selector", - token=kind_selector, context=context) - # end if - return kind - - @classmethod - def type_match(cls, line): - """Return an RE match if represents an Ftype declaration""" - match = Ftype.__itype_re.match(line.strip()) - return match - - @classmethod - def reassemble_parens(cls, propstr, errstr, context, splitstr=','): - """Return list of split by top-level instances of . - Occurrences of in character contexts or in parentheses are - ignored. - >>> Ftype.reassemble_parens("a(b, c),d,e()", 'spec', ParseContext()) - ['a(b, c)', 'd', 'e()'] - >>> Ftype.reassemble_parens("dimension(size(Grid%xlon,1),NSPC1), intent(in)", 'spec', ParseContext()) - ['dimension(size(Grid%xlon,1),NSPC1)', 'intent(in)'] - """ - var_list = list() - proplist = propstr.split(splitstr) - while len(proplist) > 0: - var = proplist.pop(0) - while var.count('(') != var.count(')'): - if len(proplist) == 0: - raise ParseSyntaxError(errstr, token=propstr, context=context) - # end if - var = var + ',' + proplist.pop(0) - # end while - var = var.strip() - if len(var) > 0: - var_list.append(var) - # end if - # end while - return var_list - - @classmethod - def parse_attr_specs(cls, propstring, context): - """Return a list of variable properties""" - properties = list() - # Remove leading comma - propstring = propstring.strip() - if propstring and (propstring[0] == ','): - propstring = propstring[1:].lstrip() - # end if - proplist = cls.reassemble_parens(propstring, 'attr_spec', context) - for prop in proplist: - prop = prop.strip().lower() - if '(' in prop: - # Strip out value from dimensions, bind, or intent - pval = prop[0:prop.index('(')].strip() - else: - pval = prop - # end if - if pval not in cls.__attr_spec: - raise ParseSyntaxError('attr_spec', token=prop, context=context) - # end if - properties.append(prop) - # end for - return properties - - @property - def typestr(self): - """ Return this FType object's type string""" - return self.__typestr - - @property - def default_kind(self): - """Return True iff this FType object is of default kind.""" - return self.__default_kind - - def kind(self): - """ Return this FType's kind string""" - return self.__kind - - @property - def type_len(self): - """ Return the length of this FType's kind string""" - return self.__match_len - - def __str__(self): - """Return a string of the declaration of the type""" - if self.default_kind: - return self.typestr - # end if - if check_fortran_intrinsic(self.typestr): - return "{}(kind={})".format(self.typestr, self.__kind) - # end if - # Derived type - return "{}({})".format(self.typestr, self.__kind) - -######################################################################## - -class FtypeCharacter(Ftype): - """FtypeCharacter is a type that represents character types - >>> FtypeCharacter.type_match('character') #doctest: +ELLIPSIS - - >>> FtypeCharacter.type_match('CHARACTER') #doctest: +ELLIPSIS - - >>> FtypeCharacter.type_match('chaRActer (len=*)') #doctest: +ELLIPSIS - - >>> FtypeCharacter.type_match('integer') - - >>> FtypeCharacter('character', ParseContext(169, 'foo.F90')).__str__() - Traceback (most recent call last): - parse_source.ParseSyntaxError: Invalid character declaration, 'character', at foo.F90:170 - >>> FtypeCharacter('character ::', ParseContext(171, 'foo.F90')).__str__() - 'character(len=1)' - >>> FtypeCharacter('CHARACTER(len=*)', ParseContext(174, 'foo.F90')).__str__() - 'CHARACTER(len=*)' - >>> FtypeCharacter('CHARACTER(len=:)', None).__str__() - 'CHARACTER(len=:)' - >>> FtypeCharacter('Character(len=512)', None).__str__() - 'Character(len=512)' - >>> FtypeCharacter('character(*)', None).__str__() - 'character(len=*)' - >>> FtypeCharacter('character(nf_file_length)', None).__str__() - 'character(len=nf_file_length)' - >>> FtypeCharacter('character(len=nf_file_length)', None).__str__() - 'character(len=nf_file_length)' - >>> FtypeCharacter('character*7', None).__str__() #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Invalid character declaration, 'character*7', at :1 - >>> FtypeCharacter('character*7,', None).__str__() - 'character(len=7)' - >>> FtypeCharacter("character (kind=kind('a')", None).__str__() #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Invalid kind_selector, 'kind=kind('a'', at :1 - >>> FtypeCharacter("character (kind=kind('a'))", None).__str__() - "character(len=1, kind=kind('a'))" - >>> FtypeCharacter("character (13, kind=kind('a'))", None).__str__() - "character(len=13, kind=kind('a'))" - >>> FtypeCharacter("character (len=13, kind=kind('a'))", None).__str__() - "character(len=13, kind=kind('a'))" - >>> FtypeCharacter("character (kind=kind('b'), len=15)", None).__str__() - "character(len=15, kind=kind('b'))" - """ - - char_re = re.compile(r"(?i)(character)\s*(\([A-Za-z0-9_,=*:\s\'\"()]+\))?") - chartrail_re = re.compile(r"\s*[,:]|\s+[A-Z]") - oldchar_re = re.compile(r"(?i)(character)\s*(\*)\s*([A-Za-z0-9_]+\s*)") - oldchartrail_re = re.compile(r"\s*[,]|\s+[A-Z]") - len_token_re = re.compile(r"(?i)([:]|[*]|[0-9]+|[A-Z][A-Z0-9_]*)$") - - @classmethod - def type_match(cls, line): - """Return an RE match if represents an FtypeCharacter - declaration""" - # Try old style first to eliminate as a possibility - match = FtypeCharacter.oldchar_re.match(line.strip()) - if match is None: - match = FtypeCharacter.char_re.match(line.strip()) - # end if - return match - - def __init__(self, line, context): - """Initialize a character type from a declaration line""" - - clen = None - kind = None # This will be interpreted as default kind - match = FtypeCharacter.type_match(line) - if match is None: - raise ParseSyntaxError("character declaration", token=line, context=context) - # end if - match_len = len(match.group(0)) - if len(match.groups()) == 3: - # We have an old style character declaration - if match.group(2) != '*': - raise ParseSyntaxError("character declaration", token=line, context=context) - # end if - if FtypeCharacter.oldchartrail_re.match(line.strip()[len(match.group(0)):]) is None: - raise ParseSyntaxError("character declaration", - token=line, context=context) - # end if - clen = match.group(3) - elif match.group(2) is not None: - # Parse attributes (strip off parentheses) - attrs = [x.strip() for x in match.group(2)[1:-1].split(',')] - if not attrs: - # Empty parentheses is not allowed - raise ParseSyntaxError("char_selector", - token=match.group(2), context=context) - # end if - if len(attrs) > 2: - # Too many attributes! - raise ParseSyntaxError("char_selector", - token=match.group(2), context=context) - # end if - if attrs[0][0:4].lower() == "kind": - # The first arg is kind, try to parse it - kind = self.parse_kind_selector(attrs[0], context=context) - # If there is a second arg, it must be of form len= - if len(attrs) == 2: - clen = self.parse_len_select(attrs[1], - context, len_optional=False) - elif len(attrs) == 2: - # We have both a len and a kind, len first - clen = self.parse_len_select(attrs[0], - context, len_optional=True) - kind = self.parse_kind_selector(attrs[1], context) - else: - # We just a len argument - clen = self.parse_len_select(attrs[0], - context, len_optional=True) - # end if - else: - # We had better check the training characters - if FtypeCharacter.chartrail_re.match(line.strip()[len(match.group(0)):]) is None: - raise ParseSyntaxError("character declaration", - token=line, context=context) - # end if - # end if - if clen is None: - clen = 1 - # end if - self.lenstr = "{}".format(clen) - super(FtypeCharacter, self).__init__(typestr_in=match.group(1), - kind_in=kind, - match_len_in=match_len, - context=context) - - def parse_len_token(self, token, context): - """Check to make sure token is a valid length identifier""" - match = FtypeCharacter.len_token_re.match(token) - if match is not None: - return match.group(1) - # end if - raise ParseSyntaxError("length type-param-value", - token=token, context=context) - # end if - - def parse_len_select(self, lenselect, context, len_optional=True): - """Parse a character type length_selector""" - largs = [x.strip() for x in lenselect.split('=')] - if len(largs) > 2: - raise ParseSyntaxError("length_selector", token=lenselect, context=context) - # end if - if (not len_optional) and ((len(largs) != 2) or (largs[0].lower() != 'len')): - raise ParseSyntaxError("length_selector when len= is required", token=lenselect, context=context) - # end if - if len(largs) == 2: - if largs[0].lower() != 'len': - raise ParseSyntaxError("length_selector", token=lenselect, context=context) - # end if - return self.parse_len_token(largs[1], context) - elif len_optional: - return self.parse_len_token(largs[0], context) - else: - raise ParseSyntaxError("length_selector when len= is required", token=lenselect, context=context) - # end if - - def kind(self): - """Return a kind metadata declaration if this Ftype object is of - a non-default kind. - Otherwise, return an empty string.""" - if self.default_kind: - kind_str = "" - else: - kind_str = ", kind={}".format(super(FtypeCharacter, self).kind()) - # end if - return "len={}{}".format(self.lenstr, kind_str) - - def __str__(self): - """Return a string of the declaration of the type - For characters, we will always print an explicit len modifier - """ - return "{}({})".format(self.typestr, self.kind()) - -######################################################################## - -class FtypeTypeDecl(Ftype): - """FtypeTypeDecl is a type that represents derived Fortran type - declarations. - >>> FtypeTypeDecl.type_match('character') - - >>> FtypeTypeDecl.type_match('type(foo)') #doctest: +ELLIPSIS - - >>> FtypeTypeDecl.type_match('class(foo)') #doctest: +ELLIPSIS - - >>> FtypeTypeDecl.class_match('class(foo)') #doctest: +ELLIPSIS - - >>> FtypeTypeDecl.type_def_line('type GFS_statein_type') - ['GFS_statein_type', None, None] - >>> FtypeTypeDecl.type_def_line('type GFS_statein_type (n, m) ') - ['GFS_statein_type', None, '(n, m)'] - >>> FtypeTypeDecl.type_def_line('type, public, extends(foo) :: GFS_statein_type') - ['GFS_statein_type', ['public', 'extends(foo)'], None] - >>> FtypeTypeDecl.type_def_line('type(foo) :: bar') - - >>> FtypeTypeDecl.type_def_line('type foo ! This is a comment') - ['foo', None, None] - """ - - __type_decl_re__ = re.compile(r"(?i)(type)\s*\(\s*([A-Z][A-Z0-9_]*)\s*\)?") - - __type_attr_spec__ = ['abstract', 'bind', 'extends', 'private', 'public'] - - __class_decl_re__ = re.compile(r"(?i)(class)\s*\(\s*([A-Z][A-Z0-9_]*)\s*\)") - - def __init__(self, line, context): - """Initialize an extended type from a declaration line""" - match = FtypeTypeDecl.type_match(line) - if match is None: - match = FtypeTypeDecl.class_match(line) - # end if - if match is None: - raise ParseSyntaxError("type declaration", - token=line, context=context) - # end if - super(FtypeTypeDecl, self).__init__(typestr_in=match.group(2), - kind_in=match.group(2), - match_len_in=len(match.group(0)), - context=context) - self.__class = match.group(1) - - @classmethod - def type_match(cls, line): - """Return an RE match if represents an FtypeTypeDecl declaration - """ - match = FtypeTypeDecl.__type_decl_re__.match(line.strip()) - # end if - return match - - @classmethod - def class_match(cls, line): - """Return an RE match if represents an FtypeTypeDecl declaration - representing the declaration of a polymorphic variable - """ - match = FtypeTypeDecl.__class_decl_re__.match(line.strip()) - # end if - return match - - @classmethod - def type_def_line(cls, line): - """Return a type information if represents the start - of a type definition. - Otherwise, return None""" - type_def = None - if not cls.type_match(line): - if '!' in line: - sline = line[0:line.index('!')].strip() - else: - sline = line.strip() - # end if - if sline.lower()[0:4] == 'type': - if '::' in sline: - elements = sline.split('::') - type_name = elements[1].strip() - type_props = [x.strip() for x in elements[0].split(',')[1:]] - else: - # Plain type decl - type_name = sline.split(' ', 1)[1].strip() - type_props = None - # end if - if '(' in type_name: - tnstr = type_name.split('(') - type_name = tnstr[0].strip() - type_params = '(' + tnstr[1].rstrip() - else: - type_params = None - # end if - type_def = [type_name, type_props, type_params] - # end if - # end if - return type_def - - def __str__(self): - """Return a printable string for this Ftype object""" - return '{}({})'.format(self.__class, self.typestr) - -######################################################################## -def ftype_factory(line, context): -######################################################################## - "Return an appropriate type object if there is a match, otherwise None" - # We have to cut off the line at the end of any possible type info - # Strip comments first (might have an = character) - if '!' in line: - line = line[0:line.index('!')].rstrip() - # end if - ppos = line.find('(') - cpos = line.find(',') - if ppos >= 0: - if 0 <= cpos < ppos: - # Whatever parentheses there are, they are not part of type - line = line[0:cpos] - else: - # Find matching right parenthesis - depth = 1 - epos = len(line) - pepos = ppos + 1 - while (depth > 0) and (pepos < epos): - if line[pepos] == '(': - depth = depth + 1 - elif line[pepos] == ')': - depth = depth - 1 - # end if - pepos = pepos + 1 - # end while - line = line[0:pepos+1] - # end if - elif cpos >= 0: - line = line[0:cpos] - # end if - tmatch = Ftype.type_match(line) - if tmatch is None: - tobj = None - else: - tobj = Ftype(line_in=line, context=context) - # end if - if tmatch is None: - tmatch = FtypeCharacter.type_match(line) - if tmatch is not None: - tobj = FtypeCharacter(line, context) - # end if - # end if - if tmatch is None: - tmatch = FtypeTypeDecl.type_match(line) - if tmatch is not None: - tobj = FtypeTypeDecl(line, context) - # end if - # end if - if tmatch is None: - tmatch = FtypeTypeDecl.class_match(line) - if tmatch is not None: - tobj = FtypeTypeDecl(line, context) - # end if - # end if - return tobj - -######################################################################## -def fortran_type_definition(line): -######################################################################## - """Return a type information if represents the start - of a type definition. - Otherwise, return None.""" - return FtypeTypeDecl.type_def_line(line) - -######################################################################## -def parse_fortran_var_decl(line, source, run_env, imports=None): -######################################################################## - """Parse a Fortran variable declaration line and return a list of - Var objects representing the variables declared on . - >>> _VAR_ID_RE.match('foo') #doctest: +ELLIPSIS - - >>> _VAR_ID_RE.match("foo()") - - >>> _VAR_ID_RE.match('foo').group(1) - 'foo' - >>> _VAR_ID_RE.match('foo').group(2) - - >>> _VAR_ID_RE.match("foo(bar)").group(1) - 'foo' - >>> _VAR_ID_RE.match("foo(bar)").group(2) - '(bar)' - >>> _VAR_ID_RE.match("foo(bar)").group(2) - '(bar)' - >>> _VAR_ID_RE.match("foo(bar, baz)").group(2) - '(bar, baz)' - >>> _VAR_ID_RE.match("foo(bar : baz)").group(2) - '(bar : baz)' - >>> _VAR_ID_RE.match("foo(bar:)").group(2) - '(bar:)' - >>> _VAR_ID_RE.match("foo(: baz)").group(2) - '(: baz)' - >>> _VAR_ID_RE.match("foo(:, :,:)").group(2) - '(:, :,:)' - >>> _VAR_ID_RE.match("foo(8)").group(2) - '(8)' - >>> _VAR_ID_RE.match("foo(::,a:b,a:,:b)").group(2) - '(::,a:b,a:,:b)' - >>> from framework_env import CCPPFrameworkEnv - >>> _DUMMY_RUN_ENV = CCPPFrameworkEnv(None, ndict={'host_files':'', \ - 'scheme_files':'', \ - 'suites':''}) - >>> parse_fortran_var_decl("integer :: foo", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('local_name') - 'foo' - >>> parse_fortran_var_decl("integer :: foo = 0", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('local_name') - 'foo' - >>> parse_fortran_var_decl("integer :: foo", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('optional') - False - >>> parse_fortran_var_decl("integer, optional :: foo", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('optional') - 'True' - >>> parse_fortran_var_decl("integer, dimension(:) :: foo", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('dimensions') - '(:)' - >>> parse_fortran_var_decl("integer, dimension(:) :: foo(bar)", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('dimensions') - '(bar)' - >>> parse_fortran_var_decl("integer, dimension(:) :: foo(:,:), baz", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('dimensions') - '(:,:)' - >>> parse_fortran_var_decl("integer, dimension(:) :: foo(:,:), baz", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[0][1].get_prop_value('dimensions') - '(:)' - >>> parse_fortran_var_decl("real (kind=kind_phys), pointer :: phii (:,:) => null() !< interface geopotential height", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('dimensions') - '(:,:)' - >>> parse_fortran_var_decl("real(kind=kind_phys), dimension(im, levs, ntrac), intent(in) :: qgrs", ParseSource('foo.F90', 'scheme', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('dimensions') - '(im, levs, ntrac)' - >>> parse_fortran_var_decl("character(len=*), intent(out) :: errmsg", ParseSource('foo.F90', 'scheme', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('local_name') - 'errmsg' - >>> parse_fortran_var_decl("character(len=512), intent(out) :: errmsg", ParseSource('foo.F90', 'scheme', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('kind') - 'len=512' - >>> parse_fortran_var_decl("real(kind_phys), intent(out) :: foo(8)", ParseSource('foo.F90', 'scheme', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('dimensions') - '(8)' - >>> parse_fortran_var_decl("real(kind_phys), intent(out) :: foo(8)", ParseSource('foo.F90', 'scheme', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_dimensions() - ['8'] - >>> parse_fortran_var_decl("character(len=*), intent(out) :: errmsg", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[1][0] - 'Syntax error: Invalid variable declaration, character(len=*), intent(out) :: errmsg, intent not allowed in module variable, in ' - >>> parse_fortran_var_decl("type(banana_t) :: bananas(0:N_FRUITS)", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[1][0] - "bananas: '0:N_FRUITS' is an invalid dimension name; integer dimension indices not supported, in " - - ## NB: Expressions (including function calls) not currently supported here - #>>> parse_fortran_var_decl("real(kind_phys), intent(out) :: foo(size(bar))", ParseSource('foo.F90', 'scheme', ParseContext()), _DUMMY_RUN_ENV)[0].get_prop_value('dimensions') - #'(size(bar))' - """ - context = source.context - sline = line.strip() - # Strip comments first - if '!' in sline: - sline = sline[0:sline.index('!')].rstrip() - # end if - tobject = ftype_factory(sline, context) - newvars = [] - errors = [] - errtyp = "Syntax error" - if tobject is not None: - varprops = sline[tobject.type_len:].strip() - def_dims = None # Default dimensions - intent = None - dimensions = None - if '::' in varprops: - elements = varprops.split('::') - varlist = elements[1].strip() - varprops = Ftype.parse_attr_specs(elements[0].strip(), context) - for prop in varprops: - if prop[0:6] == 'intent': - if source.ptype != 'scheme': - typ = source.ptype - ctx = context_string(context) - emsg1 = f"Invalid variable declaration, {sline}, " - emsg2 = f"intent not allowed in {typ} variable" - errmsg = f"{errtyp}: {emsg1}{emsg2}{ctx}" - if run_env.logger is not None: - run_env.logger.warning(errmsg) - # end if - errors.append(errmsg) - else: - intent = prop[6:].strip()[1:-1].strip() - # end if - elif prop[0:9:] == 'dimension': - dimensions = prop[9:].strip() - # end if - # end for - else: - # No attr_specs - varlist = varprops - varprops = list() - # end if - # Create Vars from these pieces - # We may need to reassemble multi-dimensional specs - var_list = Ftype.reassemble_parens(varlist, 'variable_list', context) - for var in var_list: - prop_dict = {} - if '=' in var: - # We do not care about initializers - var = var[0:var.rindex('=')].rstrip() - # end if - # Scan and gather variable pieces - inchar = None # Character context - var_len = len(var) - ploc = var.find('(') - if ploc < 0: - varname = var.strip() - dimspec = None - else: - varname = var[0:ploc].strip() - begin, end = check_balanced_paren(var) - if (begin < 0) or (end < 0): - ctx = context_string(context) - errmsg = f"{errtyp}: Invalid variable declaration, {var}{ctx}" - if run_env.logger is not None: - run_env.logger.warning(errmsg) - # end if - errors.append(errmsg) - else: - dimspec = var[begin:end+1] - # end if - # end if - prop_dict['local_name'] = varname - prop_dict['standard_name'] = unique_standard_name() - prop_dict['units'] = '' - if isinstance(tobject, FtypeTypeDecl): - prop_dict['ddt_type'] = tobject.typestr - else: - prop_dict['type'] = tobject.typestr - # end if - if tobject.kind() is not None: - prop_dict['kind'] = tobject.kind() - # end if - if 'optional' in varprops: - prop_dict['optional'] = 'True' - # end if - if 'allocatable' in varprops: - prop_dict['allocatable'] = 'True' - # end if - if intent is not None: - prop_dict['intent'] = intent - # end if - if dimspec is not None: - prop_dict['dimensions'] = dimspec - elif dimensions is not None: - prop_dict['dimensions'] = dimensions - else: - prop_dict['dimensions'] = '()' - # end if - # XXgoldyXX: I am nervous about allowing invalid Var objects here - # Also, this tends to cause an exception that ends up back here - # which is not a good idea. - try: - var = FortranVar(prop_dict, source, run_env, - fortran_imports=imports) - newvars.append(var) - except ParseSyntaxError as perr: - errors.append(str(perr)) - # end try - # end for - # No else (not a variable declaration) - # end if - return newvars, errors - -######################################################################## - -class UseStatement(object): - """Class to parse and capture information from a Fortran use statement - >>> UseStatement("use foo, only: bar").valid - True - >>> UseStatement("use foo, only: bar").module - 'foo' - >>> UseStatement("use foo, only: bar").imports - ['bar'] - >>> UseStatement("USE foo, only: bar, baz, qux").imports - ['bar', 'baz', 'qux'] - >>> UseStatement("use foo, only: bar, baz").imports - ['bar', 'baz'] - >>> UseStatement("use foo, only: bar, baz !, qux").imports - ['bar', 'baz'] - >>> UseStatement("use foo!, only: bar, baz").valid - False - >>> UseStatement("use foo!, only: bar, baz").module - 'foo' - >>> UseStatement("use foo!, only: bar, baz").imports - - """ - - __modmatch = r"use\s*("+_FORTRAN_ID+r")\s*" - __imports = r"("+_FORTRAN_ID+r"(\s*,\s*"+_FORTRAN_ID+")*)" - - __use_stmt_re = re.compile(r"(?i)"+__modmatch+r",\s*only:\s*"+__imports) - __naked_use_re = re.compile(r"(?i)use\s*("+_FORTRAN_ID+")") - - def __init__(self, line): - """Initialize a UseStatement object from .""" - match = UseStatement.__use_stmt_re.match(line.strip()) - self.__valid = match is not None - self.__module_name = None - self.__imports = None - if self.valid: - self.__module_name = match.group(1) - self.__imports = [x.strip() for x in match.group(2).split(',')] - else: - match = UseStatement.__naked_use_re.match(line.strip()) - if match: - self.__module_name = match.group(1) - # end if - # end if - - @property - def valid(self): - """Return True if this object represents a valid Fortran use statment""" - return self.__valid - - @property - def module(self): - """Return the module name if valid, otherwise, None""" - return self.__module_name - - @property - def imports(self): - """Return a list of the module's imports if valid, otherwise, None""" - return self.__imports - - @classmethod - def use_stmt_line(cls, line): - """Return True if is a Fortran use statement. - >>> UseStatement.use_stmt_line("use foo, only: bar") - True - >>> UseStatement.use_stmt_line("USE foo, only: bar, baz, qux") - True - >>> UseStatement.use_stmt_line("! use foo, only: bar") - False - """ - return UseStatement.__use_stmt_re.match(line.strip()) is not None - -######################################################################## -# Future classes -#class Ftype_type_def(FtypeTypeDecl) # Not sure about that super class -#class Fmodule_spec(object) # vars and types from a module specification part -# Fmodule_spec will contain a list of documented variables and a list of -# documented type definitions -#class Fmodule_subprog(object) # routines from a module subprogram part -#class Fmodule(object) # Info about and parsing for a Fortran module -#Fmodule will contain an Fmodule_spec and a Fmodule_subprog -######################################################################## - -######################################################################## diff --git a/scripts/fortran_tools/parse_fortran_file.py b/scripts/fortran_tools/parse_fortran_file.py deleted file mode 100644 index 9808f6c8..00000000 --- a/scripts/fortran_tools/parse_fortran_file.py +++ /dev/null @@ -1,1095 +0,0 @@ -#! /usr/bin/env python3 -""" -Tool to parse a Fortran file and return signature information -from metadata tables. -At the file level, we allow only PROGRAM blocks and MODULE blocks. -Subroutines, functions, or data are not supported outside a MODULE. -""" - -# Python library imports -import os.path -if __name__ == '__main__' and __package__ is None: - import sys - sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -# end if -# pylint: disable=wrong-import-position -import re -from collections import OrderedDict -import logging -# CCPP framework imports -from parse_tools import CCPPError, ParseInternalError, ParseSyntaxError -from parse_tools import ParseContext, ParseObject, ParseSource, PreprocStack -from parse_tools import FORTRAN_ID, context_string -from metadata_table import MetadataTable -try: - from parse_fortran import parse_fortran_var_decl, fortran_type_definition - from parse_fortran import UseStatement -except ModuleNotFoundError: - from .parse_fortran import parse_fortran_var_decl, fortran_type_definition - from .parse_fortran import UseStatement -# end try -from metavar import VarDictionary -# pylint: enable=wrong-import-position - -_COMMENT_RE = re.compile(r"!.*$") -_FIXED_COMMENT_RE = re.compile(r"(?i)([C*]|(?:[ ]{0,4}!))") -_PROGRAM_RE = re.compile(r"(?i)\s*program\s+"+FORTRAN_ID) -_ENDPROGRAM_RE = re.compile(r"(?i)\s*end\s*program\s+"+FORTRAN_ID+r"?") -_MODULE_RE = re.compile(r"(?i)\s*module\s+"+FORTRAN_ID) -_ENDMODULE_RE = re.compile(r"(?i)\s*end\s*module\s+"+FORTRAN_ID+r"?") -_CONTAINS_RE = re.compile(r"(?i)\s*contains") -_CONTINUE_RE = re.compile(r"(?i)&\s*(!.*)?$") -_FIXED_CONTINUE_RE = re.compile(r"(?i) [^0 ]") -_BLANK_RE = re.compile(r"\s+") -_ARG_TABLE_START_RE = re.compile(r"(?i)\s*![!>]\s*(?:\\section)?\s*arg_table_"+FORTRAN_ID) -_PREFIX_SPECS = [r"(?:recursive)", r"(?:pure)", r"(?:elemental)"] -_PREFIX_SPEC = r"(?:{})?\s*".format('|'.join(_PREFIX_SPECS)) -_SUBNAME_SPEC = r"subroutine\s*" -_ARGLIST_SPEC = r"\s*(?:[(]\s*([^)]*)[)])?" -_SUBROUTINE_SPEC = r"(?i)\s*"+_PREFIX_SPEC+_SUBNAME_SPEC+FORTRAN_ID+_ARGLIST_SPEC -_SUBROUTINE_RE = re.compile(_SUBROUTINE_SPEC) -_END_SUBROUTINE_RE = re.compile(r"(?i)\s*end\s*"+_SUBNAME_SPEC+FORTRAN_ID+r"?") -_USE_RE = re.compile(r"(?i)\s*use\s(?:,\s*intrinsic\s*::)?\s*only\s*:([^!]+)") -_END_TYPE_RE = re.compile(r"(?i)\s*end\s*type(?:\s+"+FORTRAN_ID+r")?") -_INTENT_STMT_RE = re.compile(r"(?i),\s*intent\s*[(]") - -######################################################################## - -def line_statements(line): - """Break up line into a list of component Fortran statements - Note, because this is a simple script, we can cheat on the - interpretation of two consecutive quote marks. - >>> line_statements('integer :: i, j') - ['integer :: i, j'] - >>> line_statements('integer :: i; real :: j') - ['integer :: i', ' real :: j'] - >>> line_statements('integer :: i ! Do not break; here') - ['integer :: i ! Do not break; here'] - >>> line_statements("write(6, *) 'This is all one statement; y''all;'") - ["write(6, *) 'This is all one statement; y''all;'"] - >>> line_statements('write(6, *) "This is all one statement; y""all;"') - ['write(6, *) "This is all one statement; y""all;"'] - >>> line_statements(" ! This is a comment statement; y'all;") - [" ! This is a comment statement; y'all;"] - >>> line_statements("!! ") - ['!! '] - >>> line_statements("real(kind_phys), intent(in) :: good_arr2(:,:)") - ['real(kind_phys), intent(in) :: good_arr2(:,:)'] - >>> line_statements("real(kind_phys), intent(in) :: bad_arr1(:,;)") - ['real(kind_phys), intent(in) :: bad_arr1(:,;)'] - >>> line_statements("real(kind_phys), intent(in), dimension(;,:) :: bad_arr2") - ['real(kind_phys), intent(in), dimension(;,:) :: bad_arr2'] - >>> line_statements("real(kind_phys), intent(in), dimension(:,;) :: bad_arr3") - ['real(kind_phys), intent(in), dimension(:,;) :: bad_arr3'] - """ - statements = list() - ind_start = 0 - ind_end = 0 - line_len = len(line) - in_single_char = False - in_double_char = False - in_paren = 0 - while ind_end < line_len: - if in_single_char: - if line[ind_end] == "'": - in_single_char = False - # end if (no else, just copy stuff in string) - elif in_double_char: - if line[ind_end] == '"': - in_double_char = False - # end if (no else, just copy stuff in string) - elif line[ind_end] == "'": - in_single_char = True - elif line[ind_end] == '"': - in_double_char = True - elif line[ind_end] == '!': - # Comment in non-character context, suck in rest of line - ind_end = line_len - 1 - elif line[ind_end] == '(': - in_paren += 1 - elif line[ind_end] == ')': - in_paren = max(in_paren - 1, 0) - elif (line[ind_end] == ';') and (in_paren < 1): - # The whole reason for this routine, the statement separator - if ind_end > ind_start: - statements.append(line[ind_start:ind_end]) - # end if - ind_start = ind_end + 1 - ind_end = ind_start - 1 - # end if (no else, other characters will be copied) - ind_end = ind_end + 1 - # end while - # Cleanup - if ind_end > ind_start: - statements.append(line[ind_start:ind_end]) - # end if - return statements - -######################################################################## - -def read_statements(pobj, statements=None): - """Retrieve the next line and break it into statements""" - while (statements is None) or (sum([len(x) for x in statements]) == 0): - nline, _ = pobj.next_line() - if nline is None: - statements = None - break - # end if - statements = line_statements(nline) - # end while - return statements - -######################################################################## -def scan_fixed_line(line, in_single_char, in_double_char, context): - """Scan a fixed-format FORTRAN line for continue indicators, continued - quotes, and comments - Return continue_in_col, in_single_char, in_double_char, - comment_col - >>> scan_fixed_line(' & line continued', False, False, ParseContext()) - (5, False, False, -1) - >>> scan_fixed_line(' & line continued"', False, True, ParseContext()) - (5, False, False, -1) - >>> scan_fixed_line(' * line continued', False, False, ParseContext()) - (5, False, False, -1) - >>> scan_fixed_line(' 1 line continued', False, False, ParseContext()) - (5, False, False, -1) - >>> scan_fixed_line('C comment line', False, False, ParseContext()) - (-1, False, False, 0) - >>> scan_fixed_line('* comment line', False, False, ParseContext()) - (-1, False, False, 0) - >>> scan_fixed_line('! comment line', False, False, ParseContext()) - (-1, False, False, 0) - >>> scan_fixed_line(' ! comment line', False, False, ParseContext()) - (-1, False, False, 1) - >>> scan_fixed_line(' ! comment line', False, False, ParseContext()) - (-1, False, False, 4) - >>> scan_fixed_line(' ! not comment line', False, False, ParseContext()) - (5, False, False, -1) - >>> scan_fixed_line('!...................................', False, False, ParseContext()) - (-1, False, False, 0) - >>> scan_fixed_line('123 x = x + 1', False, False, ParseContext()) - (-1, False, False, -1) - """ - - # Check if comment or continue statement - cmatch = _FIXED_COMMENT_RE.match(line) - is_comment = cmatch is not None - is_continue = _FIXED_CONTINUE_RE.match(line) is not None - # A few sanity checks - if (in_single_char or in_double_char) and (not is_continue): - raise ParseSyntaxError("Cannot start line in character context if not a continued line", context=context) - # endif - if in_single_char and in_double_char: - raise ParseSyntaxError("Cannot be both in an apostrophe character context and a quote character context", context=context) - - if is_continue: - continue_in_col = 5 - comment_col = -1 - index = 6 - elif is_comment: - comment_col = len(cmatch.group(1)) - 1 - continue_in_col = -1 - index = len(line.rstrip()) - else: - continue_in_col = -1 - comment_col = -1 - index = 0 - # end if - - last_ind = len(line.rstrip()) - 1 - # Process the line - while index <= last_ind: - blank = _BLANK_RE.match(line[index:]) - if blank is not None: - index = index + len(blank.group(0)) - 1 # +1 at end of loop - elif in_single_char: - if line[index:min(index+1, last_ind)] == "''": - # Embedded single quote - index = index + 1 # +1 and end of loop - elif line[index] == "'": - in_single_char = False - # end if - # end if (just ignore any other character) - elif in_double_char: - if line[index:min(index+1, last_ind)] == '""': - # Embedded double quote - index = index + 1 # +1 and end of loop - elif line[index] == '"': - in_double_char = False - # end if - # end if (just ignore any other character) - elif line[index] == "'": - # If we got here, we are not in a character context, start single - in_single_char = True - elif line[index] == '"': - # If we got here, we are not in a character context, start double - in_double_char = True - elif line[index] == '!': - # If we got here, we are not in a character context, done with line - comment_col = index - index = last_ind - # end if - index = index + 1 - # end while - - return continue_in_col, in_single_char, in_double_char, comment_col - -######################################################################## - -def scan_free_line(line, in_continue, in_single_char, in_double_char, context): - """Scan a Fortran line for continue indicators, continued quotes, and - comments - Return continue_in_col, continue_out_col, in_single_char, in_double_char, - comment_col - >>> scan_free_line("! Comment line", False, False, False, ParseContext()) - (-1, -1, False, False, 0) - >>> scan_free_line("!! ", False, False, False, ParseContext()) - (-1, -1, False, False, 0) - >>> scan_free_line("int :: index", False, False, False, ParseContext()) - (-1, -1, False, False, -1) - >>> scan_free_line("int :: inde& ! oops", False, False, False, ParseContext()) - (-1, 11, False, False, 13) - >>> scan_free_line("int :: inde&", False, False, False, ParseContext()) - (-1, 11, False, False, -1) - >>> scan_free_line("character(len=*), parameter :: foo = 'This line & not continued'", False, False, False, ParseContext()) - (-1, -1, False, False, -1) - >>> scan_free_line("character(len=*), parameter :: foo = 'This is continue line& ", False, False, False, ParseContext()) - (-1, 59, True, False, -1) - >>> scan_free_line('character(len=*), parameter :: foo = "This line & not continued"', False, False, False, ParseContext()) - (-1, -1, False, False, -1) - >>> scan_free_line('character(len=*), parameter :: foo = "This is continue line& ', False, False, False, ParseContext()) - (-1, 59, False, True, -1) - >>> scan_free_line(' & line continued"', True, False, True, ParseContext()) - (2, -1, False, False, -1) - >>> scan_free_line(' & line continued"', True, True, False, ParseContext()) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Cannot end non-continued line in a character context, in - >>> scan_free_line(" & line continued'", True, False, True, ParseContext()) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Cannot end non-continued line in a character context, in - >>> scan_free_line("int :: inde&", False, True, False, ParseContext()) - Traceback (most recent call last): - parse_source.ParseSyntaxError: Cannot start line in character context if not a continued line, in - >>> scan_free_line("int :: inde&", True, True, True, ParseContext()) - Traceback (most recent call last): - parse_source.ParseSyntaxError: Cannot be both in an apostrophe character context and a quote character context, in - """ - - # A few sanity checks - if (in_single_char or in_double_char) and (not in_continue): - raise ParseSyntaxError("Cannot start line in character context if not a continued line", context=context) - # endif - if in_single_char and in_double_char: - raise ParseSyntaxError("Cannot be both in an apostrophe character context and a quote character context", context=context) - - continue_in_col = -1 - continue_out_col = -1 - comment_col = -1 - - index = 0 - last_ind = len(line.rstrip()) - 1 - # Is first non-blank character a continue character? - if line.lstrip()[0] == '&': - if not in_continue: - raise ParseSyntaxError("Cannot begin line with continue character (&), not on continued line", context=context) - # end if - continue_in_col = line.find('&') - index = continue_in_col + 1 - # Process rest of line - while index <= last_ind: - blank = _BLANK_RE.match(line[index:]) - if blank is not None: - index = index + len(blank.group(0)) - 1 # +1 at end of loop - elif in_single_char: - if line[index:min(index+1, last_ind)] == "''": - # Embedded single quote - index = index + 1 # +1 and end of loop - elif line[index] == "'": - in_single_char = False - elif line[index] == '&': - if index == last_ind: - continue_out_col = index - # end if - # end if (just ignore any other character) - elif in_double_char: - if line[index:min(index+1, last_ind)] == '""': - # Embedded double quote - index = index + 1 # +1 and end of loop - elif line[index] == '"': - in_double_char = False - elif line[index] == '&': - if index == last_ind: - continue_out_col = index - # end if - # end if (just ignore any other character) - elif line[index] == "'": - # If we got here, we are not in a character context, start single - in_single_char = True - elif line[index] == '"': - # If we got here, we are not in a character context, start double - in_double_char = True - elif line[index] == '!': - # If we got here, we are not in a character context, done with line - comment_col = index - index = last_ind - elif line[index] == '&': - # If we got here, we are not in a character context, note continue - # First make sure this is a valid continue - match = _CONTINUE_RE.match(line[index:]) - if match is not None: - continue_out_col = index - else: - errmsg = ("Invalid continue, ampersand not followed by " - "comment character") - raise ParseSyntaxError(errmsg, context=context) - # end if - # end if - index = index + 1 - # end while - # A final check - if (in_single_char or in_double_char) and (continue_out_col < 0): - errmsg = "Cannot end non-continued line in a character context" - raise ParseSyntaxError(errmsg, context=context) - - return continue_in_col, continue_out_col, in_single_char, in_double_char, comment_col - -######################################################################## - -def read_file(filename, preproc_defs=None, logger=None): - """Read a file into an array of lines. - Preprocess lines to consolidate continuation lines. - Remove preprocessor directives and code eliminated by #if statements - Remvoved code results in blank lines, not removed lines - """ - preproc_status = PreprocStack() - if not os.path.exists(filename): - raise IOError("read_file: file, '{}', does not exist".format(filename)) - # end if - # We need special rules for fixed-form source - fixed_form = filename[-2:].lower() == '.f' - # Read all lines of the file at once - with open(filename, 'r') as file: - file_lines = file.readlines() - for index, line in enumerate(file_lines): - file_lines[index] = line.rstrip('\n').rstrip() - # end for - # end with - # create a parse object and context for this file - pobj = ParseObject(filename, file_lines) - continue_col = -1 # Active continue column - in_schar = False # Single quote character context - in_dchar = False # Double quote character context - prev_line = None - prev_line_num = -1 - curr_line, curr_line_num = pobj.curr_line() - while curr_line is not None: - # Skip empty lines and comment-only lines - skip_line = False - if len(curr_line.strip()) == 0: - skip_line = True - elif (fixed_form and - (_FIXED_COMMENT_RE.match(curr_line) is not None)): - skip_line = True - elif curr_line.lstrip()[0] == '!': - skip_line = True - # end if - if skip_line: - curr_line, curr_line_num = pobj.next_line() - continue - # end if - # Handle preproc issues - if preproc_status.process_line(curr_line, preproc_defs, pobj, logger): - pobj.write_line(curr_line_num, "") - curr_line, curr_line_num = pobj.next_line() - continue - # end if - if not preproc_status.in_true_region(): - # Special case to allow CCPP comment statements in False - # regions to find DDT and module table code - if (curr_line[0:2] != '!!') and (curr_line[0:2] != '!>'): - pobj.write_line(curr_line_num, "") - curr_line, curr_line_num = pobj.next_line() - continue - # end if - # end if - # scan the line for properties - if fixed_form: - res = scan_fixed_line(curr_line, in_schar, in_dchar, pobj) - cont_in_col, in_schar, in_dchar, comment_col = res - continue_col = cont_in_col # No warning in fixed form - cont_out_col = -1 - if (comment_col < 0) and (continue_col < 0): - # Real statement, grab the line # in case is continued - prev_line_num = curr_line_num - prev_line = None - # end if - else: - res = scan_free_line(curr_line, (continue_col >= 0), - in_schar, in_dchar, pobj) - cont_in_col, cont_out_col, in_schar, in_dchar, comment_col = res - # end if - # If in a continuation context, move this line to previous - if continue_col >= 0: - if fixed_form and (prev_line is None): - prev_line = pobj.peek_line(prev_line_num)[0:72] - # end if - if prev_line is None: - raise ParseInternalError("No prev_line to continue", - context=pobj) - # end if - sindex = max(cont_in_col+1, 0) - if fixed_form: - sindex = 6 - eindex = 72 - elif cont_out_col > 0: - eindex = cont_out_col - else: - eindex = len(curr_line) - # end if - prev_line = prev_line + curr_line[sindex:eindex] - if fixed_form: - prev_line = prev_line.rstrip() - # end if - # Rewrite the file's lines - pobj.write_line(prev_line_num, prev_line) - pobj.write_line(curr_line_num, "") - if (not fixed_form) and (cont_out_col < 0): - # We are done with this line, reset prev_line - prev_line = None - prev_line_num = -1 - # end if - # end if - continue_col = cont_out_col - if (continue_col >= 0) and (prev_line is None): - # We need to set up prev_line as it is continued - prev_line = curr_line[0:continue_col] - if not (in_schar or in_dchar): - prev_line = prev_line.rstrip() - # end if - prev_line_num = curr_line_num - # end if - curr_line, curr_line_num = pobj.next_line() - # end while - return pobj - -######################################################################## - -def parse_use_statement(statement, logger): - """Return True iff is a use statement""" - umatch = _USE_RE.match(statement) - if umatch is None: - return False - # end if - if logger: - logger.debug("use = {}".format(umatch.group(1))) - # end if - return True - -######################################################################## - -def is_dummy_argument_statement(statement): - """Return True iff is a dummy argument declaration""" - return _INTENT_STMT_RE.search(statement) is not None - -######################################################################## - -def is_comment_statement(statement): - """Return True iff is a Fortran comment""" - return statement.lstrip()[0] == '!' - -######################################################################## - -def parse_type_def(statements, type_def, mod_name, pobj, run_env, imports=None): - """Parse a type definition from and return the - remaining statements along with a MetadataTable object representing - the type's variables.""" - psrc = ParseSource(mod_name, 'ddt', pobj) - seen_contains = False - mheader = None - var_dict = VarDictionary(type_def[0], run_env) - inspec = True - errors = [] - while inspec and (statements is not None): - while len(statements) > 0: - statement = statements.pop(0) - # end program or module - pmatch = _END_TYPE_RE.match(statement) - if pmatch is not None: - # We hit the end of the type, make a header - mheader = MetadataTable(run_env, table_name_in=type_def[0], - table_type_in='ddt', - module=mod_name, var_dict=var_dict) - inspec = False - elif is_contains_statement(statement, inspec): - seen_contains = True - elif not seen_contains: - # Comment of variable - if ((not is_comment_statement(statement)) and - (not parse_use_statement(statement, run_env.logger))): - dvars, errs = parse_fortran_var_decl(statement, psrc, - run_env, - imports=imports) - errors.extend(errs) - for var in dvars: - var_dict.add_variable(var, run_env) - # end for - # end if - else: - # We are just skipping lines until the end type - pass - # end if - # end while - if inspec and (len(statements) == 0): - statements = read_statements(pobj) - # end if - # end while - return statements, mheader, errors - -######################################################################## - -def parse_preamble_data(statements, pobj, spec_name, endmatch, imports, run_env): - """Parse module variables or DDT definitions from a module preamble - or parse program variables from the beginning of a program. - Returns remaining statements, parsed metadata headers, and - any accumulated errors - """ - inspec = True - mheaders = [] - errors = [] - var_dict = VarDictionary(spec_name, run_env) - psrc = ParseSource(spec_name, 'MODULE', pobj) - active_table = None - if run_env.logger is not None: - ctx = context_string(pobj, nodir=True) - msg = "Parsing preamble variables of {}{}" - run_env.logger.debug(msg.format(spec_name, ctx)) - # end if - while inspec and (statements is not None): - while len(statements) > 0: - statement = statements.pop(0) - # end program or module - pmatch = endmatch.match(statement) - asmatch = _ARG_TABLE_START_RE.match(statement) - type_def = fortran_type_definition(statement) - if asmatch is not None: - active_table = asmatch.group(1) - elif (pmatch is not None) or is_contains_statement(statement, - inspec): - # We are done with the specification - inspec = False - # Put statement back so caller knows where we are - statements.insert(0, statement) - # Add the header (even if we found no variables) - mheader = MetadataTable(run_env, table_name_in=spec_name, - table_type_in='module', - module=spec_name, - var_dict=var_dict) - mheaders.append(mheader) - if run_env.verbose: - ctx = context_string(pobj, nodir=True) - msg = 'Adding header {}{}' - run_env.logger.debug(msg.format(mheader.table_name, ctx)) - # end if - break - elif type_def is not None: - # Put statement back so caller knows where we are - statements.insert(0, statement) - if ((active_table is not None) and - (type_def[0].lower() == active_table.lower())): - statements, ddt, errors = parse_type_def(statements, - type_def, spec_name, - pobj, run_env, - imports=imports) - if ddt is None: - ctx = context_string(pobj, nodir=True) - msg = "No DDT found at '{}'{}" - raise CCPPError(msg.format(statement, ctx)) - # end if - mheaders.append(ddt) - if run_env.verbose: - ctx = context_string(pobj, nodir=True) - msg = 'Adding DDT {}{}' - run_env.logger.debug(msg.format(ddt.table_name, ctx)) - # end if - active_table = None - else: - # We found a type definition but it is not one with - # metadata. Just parse it and throw away what is found. - _ = parse_type_def(statements, type_def, - spec_name, pobj, run_env) - # end if - elif active_table is not None: - # We should have a variable definition to add - if ((not is_comment_statement(statement)) and - (not parse_use_statement(statement, run_env.logger)) and - (active_table.lower() == spec_name.lower())): - dvars, errs = parse_fortran_var_decl(statement, - psrc, run_env) - errors.extend(errs) - for var in dvars: - var_dict.add_variable(var, run_env) - # end for - # end if - # end if (else we are not in an active table so just skip) - # end while - if inspec and (len(statements) == 0): - statements = read_statements(pobj) - # end if - # end while - return statements, mheaders, errors - -######################################################################## - -def parse_scheme_metadata(statements, pobj, spec_name, table_name, run_env): - "Parse dummy argument information from a subroutine" - psrc = None - mheader = None - var_dict = None - scheme_name = None - errors = [] - etyp = "Syntax error" - # Find the subroutine line, should be first executable statement - inpreamble = False - insub = True - seen_contains = False - if run_env.verbose: - ctx = context_string(pobj, nodir=True) - msg = "Parsing specification of {}{}" - run_env.logger.debug(msg.format(table_name, ctx)) - # end if - ctx = context_string(pobj) # Save initial context with directory - vdict = None # Initialized when we parse the subroutine arguments - while insub and (statements is not None): - while statements: - statement = statements.pop(0) - smatch = _SUBROUTINE_RE.match(statement) - esmatch = _END_SUBROUTINE_RE.match(statement) - pmatch = _ENDMODULE_RE.match(statement) - asmatch = _ARG_TABLE_START_RE.match(statement) - seen_contains = seen_contains or is_contains_statement(statement, insub) - if seen_contains: - inpreamble = False - # end if - if asmatch is not None: - # We have run off the end of something, hope that is okay - # Put this statement back for the caller to deal with - statements.insert(0, statement) - insub = False - break - # end if - if pmatch is not None: - # We have run off the end of the module, hope that is okay - pobj.leave_region('MODULE', region_name=spec_name) - insub = False - break - # end if - if smatch is not None and not seen_contains: - scheme_name = smatch.group(1) - inpreamble = scheme_name.lower() == table_name.lower() - if inpreamble: - if smatch.group(2) is not None: - smstr = smatch.group(2).strip() - if len(smstr) > 0: - smlist = smstr.strip().split(',') - else: - smlist = list() - # end if - scheme_args = [x.strip().lower() for x in smlist] - else: - scheme_args = list() - # end if - # Create a dict template with all the scheme's arguments - # in the correct order - vdict = OrderedDict() - for arg in scheme_args: - if len(arg) == 0: - errmsg = 'Empty argument{}' - raise ParseInternalError(errmsg.format(pobj)) - # end if - if arg in vdict: - ctx = context_string(pobj) - errors.append(f"Duplicate dummy argument, {arg}{ctx}") - else: - vdict[arg] = None - # end if - vdict[arg] = None - # end for - psrc = ParseSource(scheme_name, 'scheme', pobj) - # end if - elif inpreamble or seen_contains: - # Process a preamble statement (use or argument declaration) - if esmatch is not None: - inpreamble = False - seen_contains = False - insub = False - elif (inpreamble and - ((not is_comment_statement(statement)) and - (not parse_use_statement(statement, run_env)) and - is_dummy_argument_statement(statement))): - dvars, errs = parse_fortran_var_decl(statement, - psrc, run_env) - for err in errs: - # err might be an Exception instead of a string - errors.append(str(err)) - # end for - for var in dvars: - lname = var.get_prop_value('local_name').lower() - if lname in vdict: - if vdict[lname] is not None: - ctx = context_string(pobj) - errors.append(f"ERROR: Duplicate dummy argument, {lname}{ctx}") - else: - vdict[lname] = var - # end if - else: - ctx = context_string(pobj) - emsg = f"{etyp}: Invalid dummy argument, '{lname}'{ctx}" - errors.append(emsg) - # end if - # end for - # end if - # end if - # end while - if insub and (len(statements) == 0): - statements = read_statements(pobj) - # end if - # end while - # Check for missing declarations - missing = [] - if vdict is None: - errmsg = 'Subroutine, {}, not found{}' - raise CCPPError(errmsg.format(scheme_name, ctx)) - # end if - for lname in vdict.keys(): - if vdict[lname] is None: - missing.append(lname) - # end if - # end for - for lname in missing: - del vdict[lname] - # end for - if len(missing) > 0: - errmsg = f"Missing local_variables, {missing} in {scheme_name}" - errors.append(errmsg) - # end if - var_dict = VarDictionary(scheme_name, run_env, variables=vdict) - if (scheme_name is not None) and (var_dict is not None): - mheader = MetadataTable(run_env, table_name_in=scheme_name, - table_type_in='scheme', module=spec_name, - var_dict=var_dict) - # end if - return statements, mheader, errors - -######################################################################## - -def is_contains_statement(statement, in_module): - "Return True iff is an executable Fortran statement" - # Fill this in when we need to parse programs or subroutines - return in_module and (_CONTAINS_RE.match(statement.strip()) is not None) - -######################################################################## - -def duplicate_header(header, duplicate): - """Create and return an 'Duplicate header' error string""" - ctx = duplicate.start_context() - octx = header.start_context() - errmsg = 'Duplicate header, {}{}'.format(header.name, ctx) - if len(octx) > 0: - errmsg = errmsg + ', original{}'.format(octx) - # end if - return errmsg - -######################################################################## - -def parse_specification(pobj, statements, imports, run_env, mod_name=None, - prog_name=None): - """Parse specification part of a module or (sub)program. - Return the unparsed statements and a list of the parsed MetadataTable.""" - if (mod_name is not None) and (prog_name is not None): - raise ParseInternalError(" and cannot both be used") - # end if - if mod_name is not None: - spec_name = mod_name - endmatch = _ENDMODULE_RE - inmod = True - elif prog_name is not None: - spec_name = prog_name - endmatch = _ENDPROGRAM_RE - inmod = False - else: - raise ParseInternalError("One of or must be used") - # end if - if run_env.logger is not None: - ctx = context_string(pobj, nodir=True) - msg = "Parsing specification of {}{}" - run_env.logger.debug(msg.format(spec_name, ctx)) - # end if - - inspec = True - mtables = [] - errors = [] - while inspec and (statements is not None): - while len(statements) > 0: - statement = statements.pop(0) - # end program or module - pmatch = endmatch.match(statement) - asmatch = _ARG_TABLE_START_RE.match(statement) - type_def = fortran_type_definition(statement) - use_stmt = UseStatement.use_stmt_line(statement) - if pmatch is not None: - # We never found a contains statement - inspec = False - break - elif asmatch is not None: - # Put table statement back to re-read - statements.insert(0, statement) - statements, new_tbls, errors = parse_preamble_data(statements, - pobj, - spec_name, - endmatch, - imports, - run_env) - for tbl in new_tbls: - title = tbl.table_name - if title in mtables: - errors.append(duplicate_header(mtables[title], tbl)) - else: - if run_env.verbose: - ctx = tbl.start_context() - mtype = tbl.table_type - msg = "Adding metadata from {}, {}{}" - run_env.logger.debug(msg.format(mtype, title, ctx)) - # End if - mtables.append(tbl) - # end if - # end for - inspec = pobj.in_region('MODULE', region_name=mod_name) - break - elif type_def: - # We have a type definition without metadata - # Just parse it and throw away what is found. - # Put statement back so caller knows where we are - statements.insert(0, statement) - _ = parse_type_def(statements, type_def, - spec_name, pobj, run_env) - elif use_stmt: - # We have a use statement, add its imports to our set - use_obj = UseStatement(statement) - if use_obj.valid: - imports.update(use_obj.imports) - # end if - # end if - elif is_contains_statement(statement, inmod): - inspec = False - break - # end if - # end while - if inspec and (len(statements) == 0): - statements = read_statements(pobj) - # end if - # end while - return statements, mtables, errors - -######################################################################## - -def parse_program(pobj, statements, run_env): - """Parse a Fortran PROGRAM and return any leftover statements - and metadata tables encountered in the PROGRAM.""" - # The first statement should be a program statement, grab the name - pmatch = _PROGRAM_RE.match(statements[0]) - if pmatch is None: - raise ParseSyntaxError('PROGRAM statement', statements[0]) - # end if - prog_name = pmatch.group(1) - pobj.enter_region('PROGRAM', region_name=prog_name, nested_ok=False) - if run_env.logger is not None: - ctx = context_string(pobj, nodir=True) - msg = "Parsing Fortran program, {}{}" - run_env.logger.debug(msg.format(prog_name, ctx)) - # end if - # After the program name is the specification part - imports = set() - statements, mtables, errors = parse_specification(pobj, statements[1:], - imports, run_env, - prog_name=prog_name) - if errors: - raise CCPPError('\n'.join(errors)) - # end if - # We really cannot have tables inside a program's executable section - # Just read until end - statements = read_statements(pobj, statements) - inprogram = True - while inprogram and (statements is not None): - while len(statements) > 0: - statement = statements.pop(0) - # end program - pmatch = _ENDPROGRAM_RE.match(statement) - if pmatch is not None: - prog_name = pmatch.group(1) - pobj.leave_region('PROGRAM', region_name=prog_name) - inprogram = False - # end if - # end while - if inprogram and (len(statements) == 0): - statements = read_statements(pobj) - # end if - # end while - return statements, mtables - -######################################################################## - -def parse_module(pobj, statements, run_env): - """Parse a Fortran MODULE and return any leftover statements - and metadata tables encountered in the MODULE.""" - # Collect errors for more efficient error reporting - errors = [] - # Collect any imported typedef (and other) names - imports = set() - # The first statement should be a module statement, grab the name - pmatch = _MODULE_RE.match(statements[0]) - if pmatch is None: - raise ParseSyntaxError('MODULE statement', statements[0]) - # end if - mod_name = pmatch.group(1) - pobj.enter_region('MODULE', region_name=mod_name, nested_ok=False) - if run_env.verbose: - ctx = context_string(pobj, nodir=True) - msg = "Parsing Fortran module, {}{}" - run_env.logger.debug(msg.format(mod_name, ctx)) - # end if - # After the module name is the specification part - statements, mtables, errs = parse_specification(pobj, statements[1:], - imports, run_env, - mod_name=mod_name) - if errs: - errors.extend(errs) - # end if - # Look for metadata tables - statements = read_statements(pobj, statements) - inmodule = pobj.in_region('MODULE', region_name=mod_name) - active_table = None - additional_subroutines = [] - seen_contains = False - insub = False - while inmodule and (statements is not None): - while statements: - statement = statements.pop(0) - # end module - pmatch = _ENDMODULE_RE.match(statement) - asmatch = _ARG_TABLE_START_RE.match(statement) - smatch = _SUBROUTINE_RE.match(statement) - esmatch = _END_SUBROUTINE_RE.match(statement) - seen_contains = seen_contains or is_contains_statement(statement, insub) - use_stmt = UseStatement.use_stmt_line(statement) - if asmatch is not None: - active_table = asmatch.group(1) - elif pmatch is not None: - mod_name = pmatch.group(1) - pobj.leave_region('MODULE', region_name=mod_name) - inmodule = False - break - elif active_table is not None: - statements, mheader, errs = parse_scheme_metadata(statements, - pobj, - mod_name, - active_table, - run_env) - errors.extend(errs) - if mheader is not None: - title = mheader.table_name - if title in mtables: - errmsg = duplicate_header(mtables[title], mheader) - raise CCPPError(errmsg) - # end if - if run_env.verbose: - mtype = mheader.table_type - ctx = mheader.start_context() - msg = "Adding metadata from {}, {}{}" - run_env.logger.debug(msg.format(mtype, title, ctx)) - # end if - mtables.append(mheader) - # end if - active_table = None - inmodule = pobj.in_region('MODULE', region_name=mod_name) - break - elif smatch is not None and not seen_contains: - routine_name = smatch.group(1).strip() - additional_subroutines.append(routine_name) - insub = True - elif esmatch is not None and not seen_contains: - insub = False - elif esmatch is not None: - seen_contains = False - elif use_stmt: - # We have a use statement, add its imports to our set - use_obj = UseStatement(statement) - if use_obj.valid: - imports.update(use_obj.imports) - # end if - # end if - # end while - if inmodule and (statements is not None) and (len(statements) == 0): - statements = read_statements(pobj) - # end if - # end while - if errors: - raise CCPPError('\n'.join(errors)) - # end if - return statements, mtables, additional_subroutines - -######################################################################## - -def parse_fortran_file(filename, run_env): - """Parse a Fortran file and return all metadata tables found.""" - mtables = list() - pobj = read_file(filename, preproc_defs=run_env.preproc_defs, - logger=run_env.logger) - pobj.reset_pos() - curr_line, _ = pobj.curr_line() - statements = line_statements(curr_line) - while statements is not None: - if not statements: - statements = read_statements(pobj) - # end if - statement = statements.pop(0) - if _PROGRAM_RE.match(statement) is not None: - # push statement back so parse_program can use it - statements.insert(0, statement) - statements, ptables = parse_program(pobj, statements, run_env) - mtables.extend(ptables) - elif _MODULE_RE.match(statement) is not None: - # push statement back so parse_module can use it - statements.insert(0, statement) - statements, ptables, additional_routines = parse_module(pobj, statements, run_env) - mtables.extend(ptables) - # end if - if (statements is not None) and (len(statements) == 0): - statements = read_statements(pobj) - # end if - # end while - return mtables, additional_routines - -######################################################################## - -if __name__ == "__main__": - # pylint: disable=ungrouped-imports - import doctest - fail, _ = doctest.testmod() - from parse_tools import register_fortran_ddt_name - # pylint: enable=ungrouped-imports - _FPATH = '/Users/goldy/scratch/foo' - _FNAMES = ['GFS_PBL_generic.F90', 'GFS_rad_time_vary.fv3.F90', - 'GFS_typedefs.F90'] - register_fortran_ddt_name('GFS_control_type') - register_fortran_ddt_name('GFS_data_type') - for fname in _FNAMES: - fpathname = os.path.join(_FPATH, fname) - if os.path.exists(fpathname): - mh = parse_fortran_file(fpathname, preproc_defs={'CCPP':1}) - for header in mheader: - print('{}: {}'.format(fname, h)) - # end for - # end if - # end for - sys.exit(fail) -# end if diff --git a/scripts/framework_env.py b/scripts/framework_env.py deleted file mode 100644 index 2237db3e..00000000 --- a/scripts/framework_env.py +++ /dev/null @@ -1,476 +0,0 @@ -#!/usr/bin/env python3 - -""" -Module to contain the runtime options for the CCPP Framework. -Function to parse arguments to the CCPP Framework and store them in an -object which allows various framework functions to access CCPP -Framework runtime information and parameter values. -""" - -# Python library imports -import argparse -import os -from parse_tools import verbose -_EPILOG = ''' -''' - -## List of kinds in ISO_FORTRAN_ENV (that are useful in CCPP) -## Note: this is defined here instead of in fortran_tools to prevent a -## circular dependency -ISO_FORTRAN_KINDS = ['int8', 'int16', 'int32', 'int64', 'real32', 'real64', 'real128'] - -############################################################################### -class CCPPFrameworkEnv: -############################################################################### - """Object and methods to hold the runtime environment and parameter - options for the CCPP Framework""" - - def __init__(self, logger, ndict=None, verbose=0, clean=False, - host_files=None, scheme_files=None, suites=None, - preproc_directives=[], generate_docfiles=False, host_name='', - kind_types=[], use_error_obj=False, force_overwrite=False, - output_root=os.getcwd(), ccpp_datafile="datatable.xml"): - """Initialize a new CCPPFrameworkEnv object from the input arguments. - is a dict with the parsed command-line arguments (or a - dictionary created with the necessary arguments). - is a logger to be used by users of this object. - is a list defining the Fortran kind types which will be - public in ccpp_kinds.F90. - It has entries of the form: - kind_type=kind_specification[:kind_module] - where is a string defining the kind type name, - is the Fortran kind parameter name - (e.g., 'REAL64'), and is the (optional) Fortran - module that contains . If is - not specified, then must be a type defined in - ISO_FORTRAN_ENV. - It is allowed to have a duplicate entry for as long as - it does not specify a different type or module. - will be made available as a kind in ccpp_kinds.F90 - """ - emsg = '' - esep = '' - if ndict and ('verbose' in ndict): - self.__verbosity = ndict['verbose'] - del ndict['verbose'] - else: - self.__verbosity = verbose - # end if - if ndict and ('clean' in ndict): - self.__clean = ndict['clean'] - del ndict['clean'] - else: - self.__clean = clean - # end if - if ndict and ('host_files' in ndict): - self.__host_files = ndict['host_files'] - del ndict['host_files'] - if host_files and logger: - wmsg = "CCPPFrameworkEnv: Using ndict, ignoring 'host_files'" - logger.warning(wmsg) - # end if - elif host_files is None: - emsg += esep + "Error: 'host_files' list required" - esep = '\n' - else: - self.__host_files = host_files - # end if - if ndict and ('scheme_files' in ndict): - self.__scheme_files = ndict['scheme_files'] - del ndict['scheme_files'] - if scheme_files and logger: - wmsg = "CCPPFrameworkEnv: Using ndict, ignoring 'scheme_files'" - logger.warning(wmsg) - # end if - elif scheme_files is None: - emsg += esep + "Error: 'scheme_files' list required" - esep = '\n' - else: - self.__scheme_files = scheme_files - # end if - if ndict and ('suites' in ndict): - self.__suites = ndict['suites'] - del ndict['suites'] - if suites and logger: - wmsg = "CCPPFrameworkEnv: Using ndict, ignoring 'suites'" - logger.warning(wmsg) - # end if - elif suites is None: - emsg += esep + "Error: 'suites' list required" - esep = '\n' - else: - self.__suites = suites - # end if - if ndict and ('preproc_directives' in ndict): - preproc_defs = ndict['preproc_directives'] - del ndict['preproc_directives'] - else: - preproc_defs = preproc_directives - # end if - # Turn preproc_defs into a dictionary, start with a list to process - if isinstance(preproc_defs, list): - # Someone already handed us a list - preproc_list = preproc_defs - elif (not preproc_defs) or (preproc_defs == 'UNSET'): - # No preprocessor definitions - preproc_list = list() - elif ',' in preproc_defs: - # String of definitions, separated by commas - preproc_list = [x.strip() for x in preproc_defs.split(',')] - elif isinstance(preproc_defs, str): - # String of definitions, separated by spaces - preproc_list = [x.strip() for x in preproc_defs.split(' ') if x] - else: - wmsg = f"Error: Bad preproc list type, '{type_name(preproc_defs)}'" - emsg += esep + wmsg - esep = '\n' - # end if - # Turn the list into a dictionary - self.__preproc_defs = {} - for item in preproc_list: - tokens = [x.strip() for x in item.split('=', 1)] - if len(tokens) > 2: - emsg += esep + "Error: Bad preproc def, '{}'".format(item) - esep = '\n' - else: - key = tokens[0] - if key[0:2] == '-D': - key = key[2:] - # end if - if len(tokens) > 1: - value = tokens[1] - else: - value = None - # end if - self.__preproc_defs[key] = value - # end if - # end for - if ndict and ('generate_docfiles' in ndict): - self.__generate_docfiles = ndict['generate_docfiles'] - del ndict['generate_docfiles'] - else: - self.__generate_docfiles = generate_docfiles - # end if - if ndict and ('host_name' in ndict): - self.__host_name = ndict['host_name'] - del ndict['host_name'] - else: - self.__host_name = host_name - # end if - self.__generate_host_cap = self.host_name != '' - self.__kind_dict = {} - if ndict and ("kind_types" in ndict): - kind_list = ndict["kind_types"] - del ndict["kind_types"] - else: - kind_list = kind_types - # end if - # Note that the command line uses repeated calls to 'kind_type' - for kind in kind_list: - kargs = [x.strip() for x in kind.strip().split('=')] - errstr = "" - if len(kargs) != 2: - emsg += (f"{esep}Error: '{kind}' is not a valid kind specification " - "(should be of the form =)") - esep = '\n' - else: - kind_name, kind_spec = kargs - kind_specs = kind_spec.split(':') - if len(kind_specs) == 1: - errstr = self.add_kind_type(kind_name, kind_specs[0]) - elif len(kind_specs) > 2: - emsg += (f"{esep}Error: Invalid format for '{kind_name}' " - "should be [] or [, ]") - esep = '\n' - else: - errstr = self.add_kind_type(kind_name, kind_specs[0], kind_specs[1]) - # end if - if errstr: - emsg += f"{esep}{errstr}" - esep = '\n' - # end if - # end if - # end for - - # We always need a kind_phys so add a default if necessary - if "kind_phys" not in self.__kind_dict: - # Use ISO-Fortran 64-bit real - # definition for default physics kind: - self.__kind_dict['kind_phys'] = ['REAL64', 'ISO_FORTRAN_ENV'] - # end if - if ndict and ('use_error_obj' in ndict): - self.__use_error_obj = ndict['use_error_obj'] - del ndict['use_error_obj'] - else: - self.__use_error_obj = use_error_obj - # end if - if ndict and ('force_overwrite' in ndict): - self.__force_overwrite = ndict['force_overwrite'] - del ndict['force_overwrite'] - else: - self.__force_overwrite = force_overwrite - # end if - # Make sure we know where output is going - if ndict and ('output_root' in ndict): - self.__output_root = ndict['output_root'] - del ndict['output_root'] - else: - self.__output_root = output_root - # end if - self.__output_dir = os.path.abspath(self.output_root) - # Make sure we can create output database - if ndict and ('ccpp_datafile' in ndict): - self.__datatable_file = os.path.normpath(ndict['ccpp_datafile']) - del ndict['ccpp_datafile'] - else: - self.__datatable_file = ccpp_datafile - # end if - if not os.path.isabs(self.datatable_file): - self.__datatable_file = os.path.join(self.output_dir, - self.datatable_file) - # end if - self.__logger = logger - ## Check to see if anything is left in dictionary - if ndict: - for key in ndict: - emsg += esep + "Error: Unknown key in , '{}'".format(key) - esep = '\n' - # end for - # end if - # Raise an exception if any errors were found - if emsg: - raise ValueError(emsg) - # end if - - @property - def verbosity(self): - """Return the property for this CCPPFrameworkEnv object.""" - return self.__verbosity - - @property - def clean(self): - """Return the property for this CCPPFrameworkEnv object.""" - return self.__clean - - @property - def host_files(self): - """Return the property for this CCPPFrameworkEnv object.""" - return self.__host_files - - @property - def scheme_files(self): - """Return the property for this - CCPPFrameworkEnv object.""" - return self.__scheme_files - - @property - def suites(self): - """Return the property for this - CCPPFrameworkEnv object.""" - return self.__suites - - @property - def preproc_defs(self): - """Return the property for this - CCPPFrameworkEnv object.""" - return self.__preproc_defs - - @property - def generate_docfiles(self): - """Return the property for this - CCPPFrameworkEnv object.""" - return self.__generate_docfiles - - @property - def host_name(self): - """Return the property for this CCPPFrameworkEnv object.""" - return self.__host_name - - @property - def generate_host_cap(self): - """Return the property for this - CCPPFrameworkEnv object.""" - return self.__generate_host_cap - - def kind_module(self, kind_type): - """Return the Fortran module that - contains the kind specification - for kind type, , - for this CCPPFrameworkEnv object. - If there is no entry for , - return None.""" - kind_mod = None - if kind_type in self.__kind_dict: - # The kind module should always be - # the second element in the list: - kind_mod = self.__kind_dict[kind_type][1] - # end if - return kind_mod - - def kind_spec(self, kind_type): - """Return the kind specification for kind type, - for this CCPPFrameworkEnv object. - If there is no entry for , return None.""" - kind_spec = None - if kind_type in self.__kind_dict: - # The kind specification should always be - # the first element in the list: - kind_spec = self.__kind_dict[kind_type][0] - # end if - return kind_spec - - def add_kind_type(self, new_ccpp_kind, new_kind, new_module=None): - """Add to our kind dictionary. - is the name of the Fortran module that defined - is the kind name as published in ccpp_kinds.f90 - This method assumes the inputs have been parsed. - Returns None or an error string if is already in the - kinds dictionary. - """ - emsg = "" - esep = "" - # Make sure we have a valid module - if new_module == None: - if new_kind.lower() in ISO_FORTRAN_KINDS: - new_module = 'ISO_FORTRAN_ENV' - else: - emsg += (f"{esep}Error: unknown kind, '{new_kind}' " - "and no Fortran module name specified") - esep = '\n' - # end if - # end if - # Check for incompatible duplicates - if ((new_ccpp_kind in self.__kind_dict) and - ((self.kind_spec(new_ccpp_kind) != new_kind) or - (self.kind_module(new_ccpp_kind) != new_module))): - emsg += (f"{esep}Error: '{new_ccpp_kind} = [{new_kind}, {new_module}]'" - f"is an invalid duplicate. {new_ccpp_kind} " - f"is already '{str(self.__kind_dict[new_ccpp_kind])}") - esep = '\n' - else: - if new_module: - self.__kind_dict[new_ccpp_kind] = [new_kind, new_module] - # end if - # end if - return emsg - - def kind_types(self): - """Return a list of all kind types defined in this - CCPPFrameworkEnv object.""" - return self.__kind_dict.keys() - - @property - def verbose(self): - """Return true if verbose enabled for the CCPPFrameworkEnv's - logger object.""" - return (self.logger and verbose(self.logger)) - - @property - def use_error_obj(self): - """Return the property for this - CCPPFrameworkEnv object.""" - return self.__use_error_obj - - @property - def force_overwrite(self): - """Return the property for this - CCPPFrameworkEnv object.""" - return self.__force_overwrite - - @property - def output_root(self): - """Return the property for this -CCPPFrameworkEnv object.""" - return self.__output_root - - @property - def output_dir(self): - """Return the property for this CCPPFrameworkEnv object.""" - return self.__output_dir - - @property - def datatable_file(self): - """Return the property for this - CCPPFrameworkEnv object.""" - return self.__datatable_file - - @property - def logger(self): - """Return the property for this CCPPFrameworkEnv object.""" - return self.__logger - -############################################################################### -def parse_command_line(args, description, logger=None): -############################################################################### - """Create an ArgumentParser to parse and return a CCPPFrameworkEnv - object containing the command-line arguments and related quantities.""" - ap_format = argparse.RawTextHelpFormatter - parser = argparse.ArgumentParser(description=description, - formatter_class=ap_format, epilog=_EPILOG) - - parser.add_argument("--host-files", metavar='', - type=str, required=True, - help="""Comma separated list of host filenames to process -Filenames with a '.meta' suffix are treated as host model metadata files -Filenames with a '.txt' suffix are treated as containing a list of .meta -filenames""") - - parser.add_argument("--scheme-files", metavar='', - type=str, required=True, - help="""Comma separated list of scheme filenames to process -Filenames with a '.meta' suffix are treated as scheme metadata files -Filenames with a '.txt' suffix are treated as containing a list of .meta -filenames""") - - parser.add_argument("--suites", metavar='', - type=str, required=True, - help="""Comma separated list of suite definition filenames to process -Filenames with a '.xml' suffix are treated as suite definition XML files -Other filenames are treated as containing a list of .xml filenames""") - - parser.add_argument("--preproc-directives", - metavar='VARDEF1[,VARDEF2 ...]', type=str, default='', - help="Proprocessor directives used to correctly parse source files") - - parser.add_argument("--ccpp-datafile", type=str, - metavar='', - default="datatable.xml", - help="Filename for information on content generated by the CCPP Framework") - - parser.add_argument("--output-root", type=str, - metavar='', - default=os.getcwd(), - help="directory for generated files") - - parser.add_argument("--host-name", type=str, default='', - help='''Name of host model to use in CCPP API -If this option is passed, a host model cap is generated''') - - parser.add_argument("--clean", action='store_true', default=False, - help='Remove files created by this script, then exit') - - parser.add_argument("--kind-type", type=str, action='append', - metavar="kind_spec", dest="kind_types", default=list(), - help="""Data size for data (e.g., real()). -Entry in the form of = -e.g., --kind-type "kind_phys=REAL64" -Enter more than one --kind-type entry to define multiple CCPP kinds. - MUST be a valid ISO_FORTRAN_ENV type""") - - parser.add_argument("--generate-docfiles", - metavar='HTML | Latex | HTML,Latex', type=str, - help="Generate LaTeX and/or HTML documentation") - - parser.add_argument("--use-error-obj", action='store_true', default=False, - help="""Host model and caps use an error object -instead of ccpp_error_message and ccpp_error_code.""") - - parser.add_argument("--force-overwrite", action='store_true', default=False, - help="""Overwrite all CCPP-generated files, even -if unmodified""") - - parser.add_argument("--verbose", action='count', default=0, - help="Log more activity, repeat for increased output") - - pargs = parser.parse_args(args) - return CCPPFrameworkEnv(logger, vars(pargs)) diff --git a/scripts/host_cap.py b/scripts/host_cap.py deleted file mode 100644 index fb2e7012..00000000 --- a/scripts/host_cap.py +++ /dev/null @@ -1,821 +0,0 @@ -#!/usr/bin/env python3 - -""" -Parse a host-model registry XML file and return the captured variables. -""" - -# Python library imports -import logging -import os -# CCPP framework imports -from ccpp_suite import API, API_SOURCE_NAME -from ccpp_state_machine import CCPP_STATE_MACH -from constituents import ConstituentVarDict, CONST_DDT_NAME, CONST_DDT_MOD -from constituents import CONST_OBJ_STDNAME, CONST_PROP_TYPE -from ddt_library import DDTLibrary -from file_utils import KINDS_MODULE -from framework_env import CCPPFrameworkEnv -from metadata_table import MetadataTable -from metavar import Var, VarDictionary, CCPP_CONSTANT_VARS -from metavar import CCPP_LOOP_VAR_STDNAMES -from fortran_tools import FortranWriter -from parse_tools import CCPPError -from parse_tools import ParseObject, ParseSource, ParseContext, ParseSyntaxError - -############################################################################### -_HEADER = "cap for {host_model} calls to CCPP API" - -_SUBHEAD = ''' - subroutine {host_model}_ccpp_physics_{stage}({api_vars}) -''' - -_SUBFOOT = ''' - end subroutine {host_model}_ccpp_physics_{stage} -''' - -_API_SOURCE = ParseSource(API_SOURCE_NAME, "MODULE", - ParseContext(filename="host_cap.F90")) - -_API_DUMMY_RUN_ENV = CCPPFrameworkEnv(None, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - -_SUITE_NAME_VAR = Var({'local_name':'suite_name', - 'standard_name':'suite_name', - 'intent':'in', 'type':'character', - 'kind':'len=*', 'units':'', 'protected':'True', - 'dimensions':'()'}, _API_SOURCE, _API_DUMMY_RUN_ENV) - -_SUITE_PART_VAR = Var({'local_name':'suite_part', - 'standard_name':'suite_part', - 'intent':'in', 'type':'character', - 'kind':'len=*', 'units':'', 'protected':'True', - 'dimensions':'()'}, _API_SOURCE, _API_DUMMY_RUN_ENV) - -############################################################################### -# Used for creating blank dictionary -_MVAR_DUMMY_RUN_ENV = CCPPFrameworkEnv(None, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - -# Used to prevent loop substitution lookups -_BLANK_DICT = VarDictionary(API_SOURCE_NAME, _MVAR_DUMMY_RUN_ENV) - -############################################################################### -def suite_part_list(suite, stage): -############################################################################### - """Return a list of all the suite parts for this stage""" - run_stage = stage == 'run' - if run_stage: - spart_list = list() - for spart in suite.groups: - if suite.is_run_group(spart): - spart_list.append(spart) - # End if - # End for - else: - spart_list = [suite.phase_group(stage)] - # End if - return spart_list - -############################################################################### -def constituent_num_suite_subname(host_model): -############################################################################### - """Return the name of the number of suite constituents for this run - Because this is a user interface API function, the name is fixed.""" - return f"{host_model.name}_ccpp_num_suite_constituents" - -############################################################################### -def constituent_register_subname(host_model): -############################################################################### - """Return the name of the subroutine used to register the constituent - properties for this run. - Because this is a user interface API function, the name is fixed.""" - return f"{host_model.name}_ccpp_register_constituents" - -############################################################################### -def constituent_initialize_subname(host_model): -############################################################################### - """Return the name of the subroutine used to initialize the - constituents for this run. - Because this is a user interface API function, the name is fixed.""" - return f"{host_model.name}_ccpp_initialize_constituents" - -############################################################################### -def constituent_num_consts_funcname(host_model): -############################################################################### - """Return the name of the function to return the number of - constituents for this run. - Because this is a user interface API function, the name is fixed.""" - return f"{host_model.name}_ccpp_number_constituents" - -############################################################################### -def query_scheme_constituents_funcname(host_model): -############################################################################### - """Return the name of the function to return True if the standard name - passed in matches an existing constituent - Because this is a user interface API function, the name is fixed.""" - return f"{host_model.name}_ccpp_is_scheme_constituent" - -############################################################################### -def constituent_copyin_subname(host_model): -############################################################################### - """Return the name of the subroutine to copy constituent fields to the - host model. - Because this is a user interface API function, the name is fixed.""" - return f"{host_model.name}_ccpp_gather_constituents" - -############################################################################### -def constituent_copyout_subname(host_model): -############################################################################### - """Return the name of the subroutine to update constituent fields from - the host model. - Because this is a user interface API function, the name is fixed.""" - return f"{host_model.name}_ccpp_update_constituents" - -############################################################################### -def constituent_cleanup_subname(host_model): -############################################################################### - """Return the name of the subroutine to deallocate dynamic constituent - arrays - Because this is a user interface API function, the name is fixed.""" - return f"{host_model.name}_ccpp_deallocate_dynamic_constituents" - -############################################################################### -def unique_local_name(loc_name, host_model): -############################################################################### - """Create a unique local name based on the local_name property, - for a variable with standard name, . - If is an unique local name (not in ), - simply return that. If not, create one and return that.""" - new_name = host_model.find_local_name(loc_name) is not None - if new_name: - new_lname = host_model.new_internal_variable_name(prefix=loc_name) - else: - new_lname = loc_name - # end if - return new_lname - -############################################################################### -def constituent_model_object_name(host_model): -############################################################################### - """Return the variable name of the object which holds the constituent - metadata and field information.""" - hvar = host_model.find_variable(CONST_OBJ_STDNAME) - if not hvar: - raise CCPPError(f"Host model does not contain Var, {CONST_OBJ_STDNAME}") - # end if - return hvar.get_prop_value('local_name') - -############################################################################### -def suite_dynamic_constituent_array_name(host_model, suite): -############################################################################### - """Return the name of the allocatable dynamic constituent properites array""" - hstr = f"{suite}_dynamic_constituents" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_const_stdnames(host_model): -############################################################################### - """Return the name of the array of constituent standard names""" - hstr = f"{host_model.name}_model_const_stdnames" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_const_indices(host_model): -############################################################################### - """Return the name of the array of constituent field array indices""" - hstr = f"{host_model.name}_model_const_indices" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_consts(host_model): -############################################################################### - """Return the name of the function that will return a pointer to the - array of all constituents""" - hstr = f"{host_model.name}_constituents_array" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_advected_consts(host_model): -############################################################################### - """Return the name of the function that will return a pointer to the - array of advected constituents""" - hstr = f"{host_model.name}_advected_constituents_array" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_const_props(host_model): -############################################################################### - """Return the name of the array of constituent property object pointers""" - hstr = f"{host_model.name}_model_const_properties" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_const_index(host_model): -############################################################################### - """Return the name of the interface that returns the array index of - a constituent array given its standard name""" - hstr = f"{host_model.name}_const_get_index" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_consts(host_model): -############################################################################### - """Return the name of the function that will return a pointer to the - array of all constituents""" - hstr = f"{host_model.name}_constituents_array" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_advected_consts(host_model): -############################################################################### - """Return the name of the function that will return a pointer to the - array of advected constituents""" - hstr = f"{host_model.name}_advected_constituents_array" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_const_props(host_model): -############################################################################### - """Return the name of the array of constituent property object pointers""" - hstr = f"{host_model.name}_model_const_properties" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_const_index(host_model): -############################################################################### - """Return the name of the interface that returns the array index of - a constituent array given its standard name""" - hstr = f"{host_model.name}_const_get_index" - return unique_local_name(hstr, host_model) - -############################################################################### -def add_constituent_vars(cap, host_model, suite_list, run_env): -############################################################################### - """Create a DDT library containing array reference variables - for each constituent field for all suites in . - Create and return a dictionary containing an index variable for each of the - constituents as well as the variables from the DDT object. - Also, write declarations for these variables to . - Since the constituents are in a DDT (ccpp_constituent_properties_t), - create a metadata table with the required information, then parse it - to create the dictionary. - """ - # First create a MetadataTable for the constituents DDT - stdname_layer = "number_of_ccpp_constituents" - horiz_dim = "horizontal_dimension" - vert_layer_dim = "vertical_layer_dimension" - vert_interface_dim = "vertical_interface_dimension" - array_layer = "vars_layer" - tend_layer = "vars_layer_tend" - # Table preamble (leave off ccpp-table-properties header) - ddt_mdata = [ - #"[ccpp-table-properties]", - f" name = {CONST_DDT_NAME}", " type = ddt", - "[ccpp-arg-table]", - f" name = {CONST_DDT_NAME}", " type = ddt", - "[ num_layer_vars ]", - f" standard_name = {stdname_layer}", - " units = count", " dimensions = ()", " type = integer", - f"[ {array_layer} ]", - " standard_name = ccpp_constituents", - " units = none", - f" dimensions = ({horiz_dim}, {vert_layer_dim}, {stdname_layer})", - " type = real", " kind = kind_phys"] - # Add entries for each constituent (once per standard name) - const_stdnames = set() - tend_stdnames = set() - const_vars = set() - tend_vars = set() - for suite in suite_list: - if run_env.verbose: - lmsg = "Adding constituents from {} to {}" - run_env.logger.debug(lmsg.format(suite.name, host_model.name)) - # end if - scdict = suite.constituent_dictionary() - for cvar in scdict.variable_list(): - std_name = cvar.get_prop_value('standard_name') - if std_name not in const_stdnames and std_name not in tend_stdnames: - # Add a metadata entry for this constituent - # Check dimensions and figure vertical dimension - # Currently, we only support variables with first dimension, - # horizontal_dimension, and second (optional) dimension, - # vertical_layer_dimension or vertical_interface_dimension - is_tend_var = 'tendency_of' in std_name - dims = cvar.get_dimensions() - if (len(dims) < 1) or (len(dims) > 2): - emsg = "Unsupported constituent dimensions, '{}'" - dimstr = "({})".format(", ".join(dims)) - raise CCPPError(emsg.format(dimstr)) - # end if - hdim = dims[0].split(':')[-1] - if hdim != 'horizontal_dimension': - emsg = "Unsupported first constituent dimension, '{}', " - emsg += "must be 'horizontal_dimension'" - raise CCPPError(emsg.format(hdim)) - # end if - if len(dims) > 1: - vdim = dims[1].split(':')[-1] - if vdim == vert_layer_dim: - if is_tend_var: - cvar_array_name = tend_layer - else: - cvar_array_name = array_layer - # end if - else: - emsg = "Unsupported vertical constituent dimension, " - emsg += "'{}', must be '{}' or '{}'" - raise CCPPError(emsg.format(vdim, vert_layer_dim, - vert_interface_dim)) - # end if - else: - emsg = f"Unsupported 2-D variable, '{std_name}'" - raise CCPPError(emsg) - # end if - # Create an index variable for - if is_tend_var: - const_std_name = std_name.split("tendency_of_")[1] - else: - const_std_name = std_name - # end if - ind_std_name = f"index_of_{const_std_name}" - loc_name = f"{cvar_array_name}(:,:,{ind_std_name})" - ddt_mdata.append(f"[ {loc_name} ]") - ddt_mdata.append(f" standard_name = {std_name}") - units = cvar.get_prop_value('units') - ddt_mdata.append(f" units = {units}") - dimstr = f"({', '.join(dims)})" - ddt_mdata.append(f" dimensions = {dimstr}") - vtype = cvar.get_prop_value('type') - vkind = cvar.get_prop_value('kind') - ddt_mdata.append(f" type = {vtype} | kind = {vkind}") - if is_tend_var: - tend_vars.add(cvar) - tend_stdnames.add(std_name) - else: - const_vars.add(cvar) - const_stdnames.add(std_name) - # end if - - # end if - # end for - # end for - # Check that all tendency variables are valid - for tendency_variable in tend_vars: - tend_stdname = tendency_variable.get_prop_value('standard_name') - tend_const_name = tend_stdname.split('tendency_of_')[1] - found = False - # Find the corresponding constituent variable - for const_variable in const_vars: - const_stdname = const_variable.get_prop_value('standard_name') - if const_stdname == tend_const_name: - found = True - compat = tendency_variable.compatible(const_variable, run_env, is_tend=True) - if not compat: - errstr = f"Tendency variable, '{tend_stdname}'" - errstr += f", incompatible with associated state variable '{tend_const_name}'" - errstr += f". Reason: '{compat.incompat_reason}'" - raise ParseSyntaxError(errstr, token=tend_stdname, - context=tendency_variable.context) - # end if - # end if - # end for - if not found: - # error because we couldn't find the associated constituent - errstr = f"No associated state variable for tendency variable, '{tend_stdname}'" - raise ParseSyntaxError(errstr, token=tend_stdname, - context=tendency_variable.context) - # end if - # end for - # Parse this table using a fake filename - parse_obj = ParseObject(f"{host_model.name}_constituent_mod.meta", - ddt_mdata) - ddt_table = MetadataTable(run_env, parse_object=parse_obj) - ddt_lib = DDTLibrary(f"{host_model.name}_constituent_ddtlib", - run_env, ddts=ddt_table.sections()) - # A bit of cleanup - del parse_obj - del ddt_mdata - # Now, create the "host constituent module" dictionary - const_dict = VarDictionary(f"{host_model.name}_constituents", - run_env, parent_dict=host_model) - # Add the constituents object to const_dict and write its declaration - const_var = host_model.find_variable(CONST_OBJ_STDNAME) - if const_var: - const_dict.add_variable(const_var, run_env) - const_var.write_def(cap, 1, const_dict) - else: - raise CCPPError(f"Missing Var, {CONST_OBJ_STDNAME}, in host model") - # end if - ddt_lib.collect_ddt_fields(const_dict, const_var, run_env, - skip_duplicates=True) - # Declare the allocatable dynamic constituents array(s) - # One per suite - for suite in suite_list: - dyn_const_name = suite_dynamic_constituent_array_name(host_model, suite.name) - cap.write(f"type({CONST_PROP_TYPE}), allocatable, target :: {dyn_const_name}(:)", 1) - # end if - # Declare variable for the constituent standard names array - max_csname = max([len(x) for x in const_stdnames]) if const_stdnames else 0 - num_const_fields = len(const_stdnames) - cs_stdname = constituent_model_const_stdnames(host_model) - const_list = sorted(const_stdnames) - if const_list: - const_strs = ['"{}{}"'.format(x, ' '*(max_csname - len(x))) - for x in const_list] - cs_stdame_initstr = " = (/ " + ", ".join(const_strs) + " /)" - else: - cs_stdame_initstr = "" - # end if - cap.write("character(len={}) :: {}({}){}".format(max_csname, cs_stdname, - num_const_fields, - cs_stdame_initstr), 1) - # Declare variable for the constituent standard names array - array_name = constituent_model_const_indices(host_model) - cap.write("integer :: {}({}) = -1".format(array_name, num_const_fields), 1) - # Add individual variables for each index var to the const_dict - for index, std_name in enumerate(const_list): - ind_std_name = "index_of_{}".format(std_name) - ind_loc_name = "{}({})".format(array_name, index + 1) - prop_dict = {'standard_name' : ind_std_name, - 'local_name' : ind_loc_name, 'dimensions' : '()', - 'units' : 'index', 'protected' : "True", - 'type' : 'integer', 'kind' : ''} - ind_var = Var(prop_dict, _API_SOURCE, run_env) - const_dict.add_variable(ind_var, run_env) - # end for - # Add vertical dimensions for DDT call strings - pver = host_model.find_variable(standard_name=vert_layer_dim, - any_scope=False) - if pver is not None: - prop_dict = {'standard_name' : vert_layer_dim, - 'local_name' : pver.get_prop_value('local_name'), - 'units' : 'count', 'type' : 'integer', - 'protected' : 'True', 'dimensions' : '()'} - if const_dict.find_variable(standard_name=vert_layer_dim, - any_scope=False) is None: - ind_var = Var(prop_dict, _API_SOURCE, _API_DUMMY_RUN_ENV) - const_dict.add_variable(ind_var, run_env) - # end if - # end if - pver = host_model.find_variable(standard_name=vert_interface_dim, - any_scope=False) - if pver is not None: - prop_dict = {'standard_name' : vert_interface_dim, - 'local_name' : pver.get_prop_value('local_name'), - 'units' : 'count', 'type' : 'integer', - 'protected' : 'True', 'dimensions' : '()'} - if const_dict.find_variable(standard_name=vert_interface_dim, - any_scope=False) is None: - ind_var = Var(prop_dict, _API_SOURCE, run_env) - const_dict.add_variable(ind_var, run_env) - # end if - # end if - - return const_dict - -############################################################################### -def suite_part_call_list(host_model, const_dict, suite_part, subst_loop_vars, - dyn_const=False): -############################################################################### - """Return the controlled call list for . - is the constituent dictionary""" - spart_args = suite_part.call_list.variable_list(loop_vars=subst_loop_vars) - hmvars = list() # Host model to spart dummy args - if subst_loop_vars: - loop_vars = host_model.loop_vars - else: - loop_vars = None - # end if - for sp_var in spart_args: - stdname = sp_var.get_prop_value('standard_name') - sp_lname = sp_var.get_prop_value('local_name') - if sp_var.get_prop_value('type') == 'ccpp_constituent_properties_t': - if dyn_const: - hmvars.append(f"{sp_lname}={sp_lname}") - # end if - continue - # end if - var_dicts = [host_model, const_dict] - # Figure out which dictionary has the variable - for vdict in var_dicts: - hvar = vdict.find_variable(standard_name=stdname, any_scope=False) - if hvar is not None: - var_dict = vdict - break - # end if - # end for - if hvar is None: - errmsg = f"No host model variable for {stdname} in {suite_part.name}" - raise CCPPError(errmsg) - # End if - if stdname not in CCPP_CONSTANT_VARS: - lname = var_dict.var_call_string(hvar, loop_vars=loop_vars) - hmvars.append(f"{sp_lname}={lname}") - # End if - # End for - return ', '.join(hmvars) - -############################################################################### -def write_host_cap(host_model, api, module_name, output_dir, run_env): -############################################################################### - """Write an API to allow to call any configured CCPP suite""" - cap_filename = os.path.join(output_dir, '{}.F90'.format(module_name)) - if run_env.logger is not None: - msg = 'Writing CCPP Host Model Cap for {} to {}' - run_env.logger.info(msg.format(host_model.name, cap_filename)) - # End if - header = _HEADER.format(host_model=host_model.name) - with FortranWriter(cap_filename, 'w', header, module_name) as cap: - # Write module use statements - maxmod = len(KINDS_MODULE) - cap.write(' use {kinds}'.format(kinds=KINDS_MODULE), 1) - modules = host_model.variable_locations() - if modules: - mlen = max([len(x[0]) for x in modules]) - maxmod = max(maxmod, mlen) - # End if - mlen = max([len(x.module) for x in api.suites]) - maxmod = max(maxmod, mlen) - maxmod = max(maxmod, len(CONST_DDT_MOD)) - for mod in sorted(modules): - mspc = (maxmod - len(mod[0]))*' ' - cap.write("use {}, {}only: {}".format(mod[0], mspc, mod[1]), 1) - # End for - mspc = ' '*(maxmod - len(CONST_DDT_MOD)) - cap.write(f"use {CONST_DDT_MOD}, {mspc}only: {CONST_DDT_NAME}", 1) - cap.write(f"use {CONST_DDT_MOD}, {mspc}only: {CONST_PROP_TYPE}", 1) - cap.write_preamble() - max_suite_len = host_model.ddt_lib.max_mod_name_len - for suite in api.suites: - max_suite_len = max(max_suite_len, len(suite.module)) - # End for - cap.comment("Public Interfaces", 1) - # CCPP_STATE_MACH.transitions represents the host CCPP interface - for stage in CCPP_STATE_MACH.transitions(): - stmt = "public :: {host_model}_ccpp_physics_{stage}" - cap.write(stmt.format(host_model=host_model.name, stage=stage), 1) - # End for - API.declare_inspection_interfaces(cap) - # Write the host-model interfaces for constituents - reg_name = constituent_register_subname(host_model) - cap.write(f"public :: {reg_name}", 1) - init_name = constituent_initialize_subname(host_model) - cap.write(f"public :: {init_name}", 1) - numconsts_name = constituent_num_consts_funcname(host_model) - cap.write(f"public :: {numconsts_name}", 1) - queryconsts_name = query_scheme_constituents_funcname(host_model) - cap.write(f"public :: {queryconsts_name}", 1) - copyin_name = constituent_copyin_subname(host_model) - cap.write(f"public :: {copyin_name}", 1) - copyout_name = constituent_copyout_subname(host_model) - cap.write(f"public :: {copyout_name}", 1) - cleanup_name = constituent_cleanup_subname(host_model) - cap.write(f"public :: {cleanup_name}", 1) - const_array_func = constituent_model_consts(host_model) - cap.write(f"public :: {const_array_func}", 1) - advect_array_func = constituent_model_advected_consts(host_model) - cap.write(f"public :: {advect_array_func}", 1) - prop_array_func = constituent_model_const_props(host_model) - cap.write(f"public :: {prop_array_func}", 1) - const_index_func = constituent_model_const_index(host_model) - cap.write(f"public :: {const_index_func}", 1) - cap.write("", 0) - cap.write("! Private module variables", 1) - const_dict = add_constituent_vars(cap, host_model, api.suites, run_env) - cap.end_module_header() - for stage in CCPP_STATE_MACH.transitions(): - # Create a dict of local variables for stage - host_local_vars = VarDictionary(f"{host_model.name}_{stage}", - run_env) - has_dyn_consts = False - # Create part call lists - # Look for any loop-variable mismatch - for suite in api.suites: - spart_list = suite_part_list(suite, stage) - for spart in spart_list: - spart_args = spart.call_list.variable_list() - for sp_var in spart_args: - stdname = sp_var.get_prop_value('standard_name') - # Special handling for run-time constituents in register phase - if sp_var.get_prop_value('type') == 'ccpp_constituent_properties_t': - if spart.phase() == 'register': - prop_dict = {'standard_name' : sp_var.get_prop_value('standard_name'), - 'local_name' : sp_var.get_prop_value('local_name'), - 'dimensions' : '(:)', 'units' : 'none', - 'allocatable' : True, 'ddt_type' : 'ccpp_constituent_properties_t'} - newvar = Var(prop_dict, _API_SOURCE, run_env) - host_local_vars.add_variable(newvar, run_env) - has_dyn_consts = True - continue - else: - errmsg = f'ccpp_constituent_properties_t object "{stdname}" not allowed in "{spart.phase()}" phase' - raise CCPPError(errmsg) - # end if - # end if - hvar = const_dict.find_variable(standard_name=stdname, - any_scope=True) - if hvar is None: - errmsg = 'No host model variable for {} in {}' - raise CCPPError(errmsg.format(stdname, spart.name)) - # End if - # End for (loop over part variables) - # End for (loop of suite parts) - # End for (loop over suites) - if has_dyn_consts: - prop_dict = {'standard_name' : 'unused_count', - 'local_name' : 'num_dyn_consts', - 'dimensions' : '()', 'units' : 'count', - 'type' : 'integer'} - newvar = Var(prop_dict, _API_SOURCE, run_env) - host_local_vars.add_variable(newvar, run_env) - prop_dict = {'standard_name' : 'unused_index', - 'local_name' : 'const_index', - 'dimensions' : '()', 'units' : 'none', - 'type': 'integer'} - newvar = Var(prop_dict, _API_SOURCE, run_env) - host_local_vars.add_variable(newvar, run_env) - # end if - run_stage = stage == 'run' - # All interfaces need the suite name - apivars = [_SUITE_NAME_VAR] - if run_stage: - # Only the run phase needs a suite part name - apivars.append(_SUITE_PART_VAR) - # End if - # Create a list of dummy arguments with correct intent settings - callvars = host_model.call_list(stage) # Host interface dummy args - hdvars = list() - subst_dict = {} - for hvar in callvars: - protected = hvar.get_prop_value('protected') - stdname = hvar.get_prop_value('standard_name') - if stdname in CCPP_LOOP_VAR_STDNAMES: - protected = True # Cannot modify a loop variable - # End if - if protected: - subst_dict['intent'] = 'in' - else: - subst_dict['intent'] = 'inout' - # End if - hdvars.append(hvar.clone(subst_dict, - source_name=API_SOURCE_NAME)) - # End for - lnames = [x.get_prop_value('local_name') for x in apivars + hdvars] - api_vlist = ", ".join(lnames) - cap.write(_SUBHEAD.format(api_vars=api_vlist, - host_model=host_model.name, - stage=stage), 1) - # Write out any suite part use statements - for suite in api.suites: - mspc = (max_suite_len - len(suite.module))*' ' - spart_list = suite_part_list(suite, stage) - for _, spart in sorted(enumerate(spart_list)): - stmt = "use {}, {}only: {}" - cap.write(stmt.format(suite.module, mspc, spart.name), 2) - # End for - # End for - # Write out any host model DDT input var use statements - host_model.ddt_lib.write_ddt_use_statements(hdvars, cap, 2, - pad=max_suite_len) - - cap.write("", 1) - # Write out dummy argument definitions - for var in apivars: - var.write_def(cap, 2, host_model, dummy=True) - # End for - for var in hdvars: - var.write_def(cap, 2, host_model, dummy=True) - # End for - for var in host_local_vars.variable_list(): - var.write_def(cap, 2, host_model, - allocatable=var.get_prop_value('allocatable')) - # End for - cap.write('', 0) - # Write out the body clauses - errmsg_name, errflg_name = api.get_errinfo_names() - # Initialize err variables - cap.write('{errflg} = 0'.format(errflg=errflg_name), 2) - cap.write('{errmsg} = ""'.format(errmsg=errmsg_name), 2) - else_str = '' - for suite in api.suites: - stmt = "{}if (trim(suite_name) == '{}') then" - cap.write(stmt.format(else_str, suite.name), 2) - if stage == 'run': - el2_str = '' - spart_list = suite_part_list(suite, stage) - for spart in spart_list: - pname = spart.name[len(suite.name)+1:] - stmt = "{}if (trim(suite_part) == '{}') then" - cap.write(stmt.format(el2_str, pname), 3) - call_str = suite_part_call_list(host_model, const_dict, - spart, True) - cap.write("call {}({})".format(spart.name, call_str), 4) - el2_str = 'else ' - # End for - cap.write("else", 3) - emsg = "write({errmsg}, '(3a)')".format(errmsg=errmsg_name) - emsg += '"No suite part named ", ' - emsg += 'trim(suite_part), ' - emsg += '" found in suite {sname}"'.format(sname=suite.name) - cap.write(emsg, 4) - cap.write("{errflg} = 1".format(errflg=errflg_name), 4) - cap.write("end if", 3) - elif stage == 'register': - spart = suite.phase_group(stage) - dyn_const_array = suite_dynamic_constituent_array_name(host_model, suite.name) - call_str = suite_part_call_list(host_model, const_dict, spart, False, - dyn_const=True) - cap.write(f"call {suite.name}_{stage}({call_str})", 3) - cap.write(f"if ({errflg_name} /= 0) then", 3) - cap.write("return", 4) - cap.write("end if", 3) - # Allocate the suite's dynamic constituents array - size_string = "0 +" - for var in host_local_vars.variable_list(): - vtype = var.get_prop_value('type') - if vtype == 'ccpp_constituent_properties_t': - local_name = var.get_prop_value('local_name') - size_string += f"size({local_name}) +" - # end if - # end for - if not has_dyn_consts: - cap.comment("Suite does not return dynamic constituents; allocate to zero", 3) - # end if - cap.write(f"allocate({dyn_const_array}({size_string[:-1]}))", 3) - if has_dyn_consts: - cap.comment("Pack the suite-level dynamic, run-time constituents array", 3) - cap.write("num_dyn_consts = 0", 3) - for var in host_local_vars.variable_list(): - vtype = var.get_prop_value('type') - if vtype != 'ccpp_constituent_properties_t': - continue - # end if - local_name = var.get_prop_value('local_name') - cap.write(f"do const_index = 1, size({local_name})", 3) - cap.write(f"{dyn_const_array}(num_dyn_consts + const_index) = {local_name}(const_index)", 4) - cap.write("end do", 3) - cap.write(f"num_dyn_consts = num_dyn_consts + size({local_name})", 3) - cap.write(f"deallocate({local_name})", 3) - # end for - - else: - spart = suite.phase_group(stage) - call_str = suite_part_call_list(host_model, const_dict, - spart, False) - stmt = "call {}_{}({})" - cap.write(stmt.format(suite.name, stage, call_str), 3) - # End if - else_str = 'else ' - # End for - cap.write("else", 2) - emsg = "write({errmsg}, '(3a)')".format(errmsg=errmsg_name) - emsg += '"No suite named ", ' - emsg += 'trim(suite_name), "found"' - cap.write(emsg, 3) - cap.write("{errflg} = 1".format(errflg=errflg_name), 3) - cap.write("end if", 2) - cap.write(_SUBFOOT.format(host_model=host_model.name, - stage=stage), 1) - # End for - # Write the API inspection routines (e.g., list of suites) - api.write_inspection_routines(cap) - # Write the constituent initialization interfaces - err_vars = host_model.find_error_variables() - const_obj_name = constituent_model_object_name(host_model) - cap.write("", 0) - const_names_name = constituent_model_const_stdnames(host_model) - const_indices_name = constituent_model_const_indices(host_model) - dyn_const_names = [suite_dynamic_constituent_array_name(host_model, suite.name) for suite in api.suites] - ConstituentVarDict.write_host_routines(cap, host_model, reg_name, init_name, - numconsts_name, queryconsts_name, - copyin_name, copyout_name, - cleanup_name, - const_obj_name, - dyn_const_names, - const_names_name, - const_indices_name, - const_array_func, - advect_array_func, - prop_array_func, - const_index_func, - api.suites, - err_vars) - # End with - return cap_filename - -############################################################################### - -if __name__ == "__main__": - from parse_tools import init_log, set_log_to_null - _LOGGER = init_log('host_registry') - set_log_to_null(_LOGGER) - # Run doctest - # pylint: disable=ungrouped-imports - import doctest - import sys - # pylint: enable=ungrouped-imports - fail, _ = doctest.testmod() - sys.exit(fail) -# end if diff --git a/scripts/host_model.py b/scripts/host_model.py deleted file mode 100644 index c3beb447..00000000 --- a/scripts/host_model.py +++ /dev/null @@ -1,344 +0,0 @@ -#!/usr/bin/env python3 - -""" -Parse a host-model registry XML file and return the captured variables. -""" - -# CCPP framework imports -from constituents import CONST_DDT_NAME, CONST_PROP_TYPE, CONST_OBJ_STDNAME -from metavar import Var, VarDictionary -from ddt_library import VarDDT, DDTLibrary -from parse_tools import ParseContext, ParseSource, CCPPError, ParseInternalError -from parse_tools import context_string, registered_fortran_ddt_name -from parse_tools import FORTRAN_SCALAR_REF_RE - -############################################################################### -class HostModel(VarDictionary): - """Class to hold the data from a host model""" - - def __init__(self, meta_tables, name_in, run_env): - """Initialize this HostModel object. - is a dictionary of parsed host metadata tables. - - dictionary key is title of metadata argtable - is the name for this host model. - is the CCPPFrameworkEnv object for this framework run. - """ - self.__name = name_in - self.__var_locations = {} # Local name to module map - self.__loop_vars = None # Loop control vars in interface calls - self.__used_variables = None # Local names which have been requested - self.__deferred_finds = None # Used variables that were missed at first - self.__run_env = run_env - # First, process DDT headers - meta_headers = list() - for sect in [x.sections() for x in meta_tables.values()]: - meta_headers.extend(sect) - # end for - # Initialize our dictionaries - # Initialize variable dictionary - super().__init__(self.name, run_env) - ddt_headers = [d for d in meta_headers if d.header_type == 'ddt'] - self.__ddt_lib = DDTLibrary('{}_ddts'.format(self.name), run_env, - ddts=ddt_headers) - self.__ddt_dict = VarDictionary("{}_ddt_vars".format(self.name), - run_env, parent_dict=self) - del ddt_headers - # Now, process the code headers by type - self.__metadata_tables = meta_tables - for header in [h for h in meta_headers if h.header_type != 'ddt']: - title = header.title - if run_env.logger is not None: - msg = 'Adding {} {} to host model' - run_env.logger.debug(msg.format(header.header_type, title)) - # End if - if header.header_type == 'module': - # Set the variable modules - modname = header.title - for var in header.variable_list(): - self.add_variable(var, run_env) - lname = var.get_prop_value('local_name') - self.__var_locations[lname] = modname - self.ddt_lib.check_ddt_type(var, header, lname=lname) - if var.is_ddt(): - self.ddt_lib.collect_ddt_fields(self.__ddt_dict, var, - run_env) - # End if - # End for - elif header.header_type == 'host': - if self.__name is None: - # Grab the first host name we see - self.__name = header.name - # End if - for var in header.variable_list(): - self.add_variable(var, run_env) - self.ddt_lib.check_ddt_type(var, header) - if var.is_ddt(): - self.ddt_lib.collect_ddt_fields(self.__ddt_dict, var, - run_env) - # End if - # End for - loop_vars = header.variable_list(std_vars=False, - loop_vars=True, consts=False) - loop_vars.extend(self.__ddt_dict.variable_list(std_vars=False, - loop_vars=True, - consts=False)) - if loop_vars: - # loop_vars are part of the host-model interface call - # at run time. As such, they override the host-model - # array dimensions. - self.__loop_vars = VarDictionary(self.name, run_env) - # End if - for hvar in loop_vars: - std_name = hvar.get_prop_value('standard_name') - if std_name not in self.__loop_vars: - self.__loop_vars.add_variable(hvar, run_env) - else: - ovar = self.__loop_vars[std_name] - ctx1 = context_string(ovar.context) - ctx2 = context_string(hvar.context) - lname1 = ovar.get_prop_value('local_name') - lname2 = hvar.get_prop_value('local_name') - errmsg = ("Duplicate host loop var for {n}:\n" - " Dup: {l1}{c1}\n Orig: {l2}{c2}") - raise CCPPError(errmsg.format(n=self.name, - l1=lname1, c1=ctx1, - l2=lname2, c2=ctx2)) - # End if - # End for - else: - errmsg = "Invalid host model metadata header type, {} ({}){}" - errmsg += "\nType must be 'module' or 'host'" - ctx = context_string(header.context) - raise CCPPError(errmsg.format(header.title, - header.header_type, ctx)) - # End if - # End while - if self.name is None: - errmsg = 'No name found for host model, add a host metadata entry' - raise CCPPError(errmsg) - # End if - # Add in the constituents object - if registered_fortran_ddt_name(CONST_PROP_TYPE): - prop_dict = {'standard_name' : CONST_OBJ_STDNAME, - 'local_name' : self.constituent_model_object_name(), - 'dimensions' : '()', 'units' : "None", - 'ddt_type' : CONST_DDT_NAME, 'target' : 'True'} - host_source = ParseSource(self.ccpp_cap_name(), "MODULE", - ParseContext(filename=f"{self.ccpp_cap_name()}.F90")) - const_var = Var(prop_dict, host_source, run_env) - self.add_variable(const_var, run_env) - lname = const_var.get_prop_value('local_name') - self.__var_locations[lname] = self.ccpp_cap_name() - self.ddt_lib.collect_ddt_fields(self.__ddt_dict, const_var, run_env) - # end if - # Finally, turn on the use meter so we know which module variables - # to 'use' in a host cap. - self.__used_variables = set() # Local names which have been requested - self.__deferred_finds = set() # Used variables that were missed at first - - @property - def name(self): - """Return the host model name""" - return self.__name - - @property - def loop_vars(self): - """Return this host model's loop variables""" - return self.__loop_vars - - @property - def ddt_lib(self): - """Return this host model's DDT library""" - return self.__ddt_lib - - @property - def constituent_module(self): - """Return the name of host model constituent module""" - return f"{self.name}_ccpp_constituents" - - def argument_list(self, loop_vars=True): - """Return a string representing the host model variable arg list""" - args = [v.call_string(self) - for v in self.variable_list(loop_vars=loop_vars, consts=False)] - return ', '.join(args) - - def metadata_tables(self): - """Return a copy of this host models metadata tables""" - return dict(self.__metadata_tables) - - def host_variable_module(self, local_name): - """Return the module name for a host variable""" - if local_name in self.__var_locations: - return self.__var_locations[local_name] - # End if - return None - - def variable_locations(self): - """Return a set of module-variable and module-type pairs. - These represent the locations of all host model data with a listed - source location (variables with no source or for which the - source is the CCPP host cap are omitted).""" - varset = set() - lnames = self.prop_list('local_name') - # Attempt to realize deferred lookups - if self.__deferred_finds is not None: - for std_name in list(self.__deferred_finds): - var = self.find_variable(standard_name=std_name) - if var is not None: - self.__deferred_finds.remove(std_name) - # End if - # End for - # End if - # Now, find all the used module variables - cap_modname = self.ccpp_cap_name() - for name in lnames: - module = self.host_variable_module(name) - used = self.__used_variables and (name in self.__used_variables) - if module and used and (module != cap_modname): - varset.add((module, name)) - # No else, either no module or a zero-length module name - # End if - # End for - return varset - - def find_variable(self, standard_name=None, source_var=None, - any_scope=False, clone=None, - search_call_list=False, loop_subst=False): - """Return the host model variable matching or None - If is True, substitute a begin:end range for an extent. - """ - my_var = super().find_variable(standard_name=standard_name, - source_var=source_var, - any_scope=any_scope, clone=clone, - search_call_list=search_call_list, - loop_subst=loop_subst) - if my_var is None: - # Check our DDT library - if standard_name is None: - if source_var is None: - emsg = ("One of or " + - "must be passed.") - raise ParseInternalError(emsg) - # end if - standard_name = source_var.get_prop_value('standard_name') - # end if - # Since we are the parent of the DDT library, only check that dict - my_var = self.__ddt_dict.find_variable(standard_name=standard_name, - any_scope=False) - # End if - if loop_subst: - if my_var is None: - my_var = self.find_loop_subst(standard_name) - # End if - if my_var is not None: - # If we get here, the host does not have the requested - # variable but does have a replacement set. Create a new - # variable to use to send to suites. - ##XXgoldyXX: This cannot be working since find_loop_subst - ## returns a tuple - new_name = self.new_internal_variable_name(prefix=self.name) - ctx = ParseContext(filename='host_model.py') - new_var = my_var.clone(new_name, source_name=self.name, - source_type="HOST", - context=ctx) - self.add_variable(new_var, self.__run_env) - my_var = new_var - # End if - # End if - if my_var is None: - if self.__deferred_finds is not None: - self.__deferred_finds.add(standard_name) - # End if - elif self.__used_variables is not None: - lname = my_var.get_prop_value('local_name') - # Try to add any index references (should be method?) - imatch = FORTRAN_SCALAR_REF_RE.match(lname) - if imatch is not None: - vdims = [x.strip() for x in imatch.group(2).split(',') - if ':' not in x] - for vname in vdims: - _ = self.find_variable(standard_name=vname) - # End for - # End if - if isinstance(my_var, VarDDT): - lname = my_var.get_parent_prop('local_name') - # End if - self.__used_variables.add(lname) - # End if - return my_var - - def add_variable(self, newvar, run_env, exists_ok=False, gen_unique=False, - adjust_intent=False): - """Add if it does not conflict with existing entries. - For the host model, this includes entries in used DDT variables. - If is True, attempting to add an identical copy is okay. - If is True, a new local_name will be created if a - local_name collision is detected. - if is True, adjust conflicting intents to inout.""" - standard_name = newvar.get_prop_value('standard_name') - cvar = self.find_variable(standard_name=standard_name, any_scope=False) - if cvar is None: - # Check the DDT dictionary - cvar = self.__ddt_dict.find_variable(standard_name=standard_name, - any_scope=False) - # end if - if cvar and (not exists_ok): - emsg = "Attempt to add duplicate host model variable, {}{}." - emsg += "\nVariable originally defined{}" - ntx = context_string(newvar.context) - ctx = context_string(cvar.context) - raise CCPPError(emsg.format(standard_name, ntx, ctx)) - # end if - # No collision, proceed normally - super().add_variable(newvar=newvar, run_env=run_env, - exists_ok=exists_ok, gen_unique=gen_unique, - adjust_intent=False) - - def add_host_variable_module(self, local_name, module, logger=None): - """Add a module name location for a host variable""" - if local_name not in self.__var_locations: - if logger is not None: - emsg = 'Adding variable, {}, from module, {}' - logger.debug(emsg.format(local_name, module)) - # End if - self.__var_locations[local_name] = module - else: - emsg = "Host variable, {}, already located in module" - raise CCPPError(emsg.format(self.__var_locations[local_name])) - # End if - - def call_list(self, phase): - "Return the list of variables passed by the host model to the host cap" - hdvars = list() - loop_vars = phase == 'run' - for hvar in self.variable_list(loop_vars=loop_vars, consts=False): - lname = hvar.get_prop_value('local_name') - if self.host_variable_module(lname) is None: - hdvars.append(hvar) - # End if - # End for - return hdvars - - def constituent_model_object_name(self): - """Return the variable name of the object which holds the constituent - metadata and field information.""" - return "{}_constituents_obj".format(self.name) - - def ccpp_cap_name(self): - """Return the name of the CCPP host model cap module name.""" - return f"{self.name}_ccpp_cap" - -############################################################################### - -if __name__ == "__main__": - # pylint: disable=ungrouped-imports - from parse_tools import init_log, set_log_to_null - import doctest - import sys - # pylint: enable=ungrouped-imports - _LOGGER = init_log('host_registry') - set_log_to_null(_LOGGER) - # First, run doctest - fail, _ = doctest.testmod() - sys.exit(fail) -# end if diff --git a/scripts/metadata2html.py b/scripts/metadata2html.py deleted file mode 100755 index 7e4a540d..00000000 --- a/scripts/metadata2html.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import logging -import importlib -import os -import sys - -# CCPP framework imports -from common import CCPP_INTERNAL_VARIABLE_DEFINITON_FILE -from parse_checkers import registered_fortran_ddt_names -from parse_tools import init_log, set_log_level -from metadata_table import MetadataTable, parse_metadata_file -from framework_env import CCPPFrameworkEnv - -############################################################################### -# Set up the command line argument parser and other global variables # -############################################################################### - -parser = argparse.ArgumentParser() -method = parser.add_mutually_exclusive_group(required=True) -method.add_argument('--config', '-c', action='store', - help='path to CCPP prebuild configuration file') -method.add_argument('--metafile', '-m', action='store', - help='name of metadata file to convert (requires -o)') -parser.add_argument('--outputdir', '-o', action='store', - help='directory where to write the html files', - required='--metafile' in sys.argv or '-m' in sys.argv) - -# List and order of variable attributes to output to HTML -ATTRIBUTES = [ 'local_name', 'standard_name', 'long_name', 'units', - 'type', 'dimensions', 'kind', 'intent' ] - -############################################################################### -# Functions and subroutines # -############################################################################### - -def parse_arguments(): - """Parse command line arguments.""" - args = parser.parse_args() - config = args.config - filename = args.metafile - outdir = args.outputdir - return (config, filename, outdir) - -def import_config(configfile, logger): - """Import the configuration from a given configuration file""" - - if not os.path.isfile(configfile): - raise Exception("Configuration file {0} not found".format(configfile)) - - # Import the host-model specific CCPP prebuild config; - # split into path and module name for import - configpath = os.path.abspath(os.path.dirname(configfile)) - configmodule = os.path.splitext(os.path.basename(configfile))[0] - sys.path.append(configpath) - ccpp_prebuild_config = importlib.import_module(configmodule) - - # Get the base directory for running metadata2html.py from - # the default build directory value in the CCPP prebuild config - basedir = os.path.join(os.getcwd()) - logger.info('Relative path to CCPP directory from CCPP prebuild config: {}'.format( - ccpp_prebuild_config.DEFAULT_BUILD_DIR)) - - config = {} - # Definitions in host-model dependent CCPP prebuild config script - config['variable_definition_files'] = ccpp_prebuild_config.VARIABLE_DEFINITION_FILES - config['scheme_files'] = ccpp_prebuild_config.SCHEME_FILES - # Add model-independent, CCPP-internal variable definition files - config['variable_definition_files'].append(CCPP_INTERNAL_VARIABLE_DEFINITON_FILE) - # Output directory for converted metadata tables - config['metadata_html_output_dir'] = ccpp_prebuild_config.METADATA_HTML_OUTPUT_DIR.format(build_dir=basedir) - - return config - -def get_metadata_files_from_config(config, logger): - """Create a list of metadata filenames for a CCPP prebuild configuration""" - filenames = [] - for sourcefile in config['variable_definition_files'] + config['scheme_files']: - metafile = os.path.splitext(sourcefile)[0]+'.meta' - if os.path.isfile(metafile): - filenames.append(metafile) - else: - # DH* Warn for now, raise exception later when - # old metadata format is no longer supported - logger.warn("Metadata file {} for source file {} not found, assuming old metadata format".format( - metafile, sourcefile)) - return filenames - -def get_output_directory_from_config(config, logger): - """Return the html output directory for a CCPP prebuild configuration""" - outdir = config['metadata_html_output_dir'] - if not os.path.isdir(outdir): - raise Exception("Output directory {} for converted metadata tables does not exist".format(outdir)) - return outdir - -def convert_to_html(filename_in, outdir, logger, run_env): - """Convert a metadata file into html (one html file for each table)""" - if not os.path.isfile(filename_in): - raise Exception("Metadata file {} not found".format(filename_in)) - logger.info("Converting file {} to HTML".format(filename_in)) - metadata_headers = parse_metadata_file(filename_in, - known_ddts=registered_fortran_ddt_names(), - run_env=run_env) - for metadata_header in metadata_headers: - for metadata_section in metadata_header.sections(): - filename_out = metadata_section.to_html(outdir, ATTRIBUTES) - if filename_out: - logger.info(" ... wrote {}".format(filename_out)) - -def main(): - # Initialize logging - logger = init_log('metadata2html') - set_log_level(logger, logging.INFO) - run_env = CCPPFrameworkEnv(logger, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - - # Convert metadata file - (configfile, filename, outdir) = parse_arguments() - if configfile: - config = import_config(configfile, logger) - filenames = get_metadata_files_from_config(config, logger) - outdir = get_output_directory_from_config(config, logger) - for filename in filenames: - convert_to_html(filename, outdir, logger, run_env) - else: - convert_to_html(filename, outdir, logger, run_env) - -if __name__ == '__main__': - main() diff --git a/scripts/metadata_parser.py b/scripts/metadata_parser.py deleted file mode 100755 index 0b25538f..00000000 --- a/scripts/metadata_parser.py +++ /dev/null @@ -1,790 +0,0 @@ -#!/usr/bin/env python3 - -import collections -import logging -import os -import re -import subprocess -import sys -from xml.etree import ElementTree as ET - -from common import encode_container, CCPP_STAGES -from common import CCPP_ERROR_CODE_VARIABLE, CCPP_ERROR_MSG_VARIABLE -from common import insert_plus_sign_for_positive_exponents -from mkcap import Var - -sys.path.append(os.path.join(os.path.split(__file__)[0], 'fortran_tools')) -from parse_fortran import FtypeTypeDecl -from parse_checkers import registered_fortran_ddt_names -from parse_tools import init_log -from metadata_table import MetadataTable, parse_metadata_file -from framework_env import CCPPFrameworkEnv - -_API_LOGGING = init_log('metadata_parser') -_DUMMY_RUN_ENV = CCPPFrameworkEnv(_API_LOGGING, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - -# Output: This routine converts the argument tables for all subroutines / typedefs / kind / module variables -# into dictionaries suitable to be used with ccpp_prebuild.py (which generates the fortran code for the caps) - -# Items in this dictionary are used for checking valid entries in metadata tables. For columns with no keys/keys -# commented out, no check is performed. This is the case for 'type' and 'kind' right now, since models use their -# own derived data types and kind types. -VALID_ITEMS = { - 'header' : ['local_name', 'standard_name', 'long_name', 'units', 'rank', 'type', 'kind', 'intent'], - #'type' : ['character', 'integer', 'real', ...], - #'kind' : ['default', 'kind_phys', ...], - 'intent' : ['none', 'in', 'out', 'inout'], - } - -# Mandatory variables that every scheme needs to have -CCPP_MANDATORY_VARIABLES = { - CCPP_ERROR_MSG_VARIABLE : Var(local_name = 'errmsg', - standard_name = CCPP_ERROR_MSG_VARIABLE, - long_name = 'error message for error handling in CCPP', - units = 'none', - type = 'character', - dimensions = [], - rank = '', - kind = 'len=*', - intent = 'out', - active = 'T', - ), - CCPP_ERROR_CODE_VARIABLE : Var(local_name = 'ierr', - standard_name = CCPP_ERROR_CODE_VARIABLE, - long_name = 'error code for error handling in CCPP', - units = '1', - type = 'integer', - dimensions = [], - rank = '', - kind = '', - intent = 'out', - active = 'T', - ), - } - -# Save metadata to avoid repeated parsing of type/variable definition files -NEW_METADATA_SAVE = {} - -############################################################################### - -def merge_dictionaries(x, y): - """Merges two metadata dictionaries. For each list of elements - (variables = class Var in mkcap.py) in one dictionary, we know - that all entries are compatible. If one or more elements exist - in both x and y, we therefore have to test compatibility of - one of the items in each dictionary only.""" - z = collections.OrderedDict() - x_keys = sorted(x.keys()) - y_keys = sorted(y.keys()) - z_keys = sorted(list(set(x_keys + y_keys))) - for key in z_keys: - z[key] = collections.OrderedDict() - if key in x_keys and key in y_keys: - # Metadata dictionaries containing lists of variables of type Var for each key=standard_name - if isinstance(x[key][0], Var): - # We know that all entries within each dictionary are compatible; - # we need to test compatibility of one of the items in each only. - if not x[key][0].compatible(y[key][0]): - raise Exception('Incompatible entries in metadata for variable {0}:\n'.format(key) +\ - ' {0}\n'.format(x[key][0].print_debug()) +\ - 'vs. {0}'.format(y[key][0].print_debug())) - z[key] = x[key] + y[key] - # Physics set dictionaries containing lists of physics sets of type string for each key=standard_name - elif type(x[key][0]) is str: - z[key] = list(set(x[key] + y[key])) - else: - raise Exception("x[key][0] is of unsupported type", type(x[key][0])) - elif key in x_keys: - z[key] = x[key] - elif key in y_keys: - z[key] = y[key] - return z - -def read_new_metadata(filename, module_name, table_name, scheme_name = None, subroutine_name = None): - """Read metadata in new format and convert output to ccpp_prebuild metadata dictionary""" - if not os.path.isfile(filename): - raise Exception("New metadata file {0} not found".format(filename)) - - # Save metadata, because this routine new_metadata - # is called once for every table in that file - if filename in NEW_METADATA_SAVE.keys(): - new_metadata_headers = NEW_METADATA_SAVE[filename] - else: - new_metadata_headers = parse_metadata_file(filename, known_ddts=registered_fortran_ddt_names(), - run_env=_DUMMY_RUN_ENV) - NEW_METADATA_SAVE[filename] = new_metadata_headers - - # Record dependencies for the metadata table (only applies to schemes) - dependencies = [] - - # Convert new metadata for requested table to old metadata dictionary - metadata = collections.OrderedDict() - for new_metadata_header in new_metadata_headers: - for metadata_section in new_metadata_header.sections(): - metadata_section_title = metadata_section.title.lower() - # Module or DDT tables - if not scheme_name: - # Module property tables - if not metadata_section_title == table_name: - # Skip this table, since it is not requested right now - continue - - # Distinguish between module argument tables and DDT argument tables - if metadata_section_title == module_name: - container = encode_container(module_name) - else: - container = encode_container(module_name, metadata_section_title) - - # Add to dependencies - if new_metadata_header.dependencies_path: - dependencies += [ os.path.join(new_metadata_header.dependencies_path, x) for x in new_metadata_header.dependencies] - else: - dependencies += new_metadata_header.dependencies - else: - # Scheme property tables - if not metadata_section_title == table_name: - # Skip this table, since it is not requested right now - continue - - container = encode_container(module_name, scheme_name, table_name) - - # Add to dependencies - if new_metadata_header.dependencies_path: - dependencies += [ os.path.join(new_metadata_header.dependencies_path, x) for x in new_metadata_header.dependencies] - else: - dependencies += new_metadata_header.dependencies - - for new_var in metadata_section.variable_list(): - standard_name = new_var.get_prop_value('standard_name') - # DH* 2020-05-26 - # Legacy extension for inconsistent metadata (use of horizontal_dimension versus horizontal_loop_extent). - # Since horizontal_dimension and horizontal_loop_extent have the same attributes (otherwise it doesn't - # make sense), we swap the standard name and add a note to the long name - 2021-05-26: this is now an error. - legacy_note = '' - if standard_name == 'horizontal_loop_extent' and scheme_name and \ - (table_name.endswith("_init") or table_name.endswith("_finalize")): - #logging.warn("Legacy extension - replacing variable 'horizontal_loop_extent'" + \ - # " with 'horizontal_dimension' in table {}".format(table_name)) - #standard_name = 'horizontal_dimension' - #legacy_note = ' replaced by horizontal dimension (legacy extension)' - raise Exception("Legacy extension DISABLED: replacing variable 'horizontal_loop_extent'" + \ - " with 'horizontal_dimension' in table {}".format(table_name)) - elif standard_name == 'horizontal_dimension' and scheme_name and table_name.endswith("_run"): - #logging.warn("Legacy extension - replacing variable 'horizontal_dimension'" + \ - # " with 'horizontal_loop_extent' in table {}".format(table_name)) - #standard_name = 'horizontal_loop_extent' - #legacy_note = ' replaced by horizontal loop extent (legacy extension)' - raise Exception("Legacy extension DISABLED: replacing variable 'horizontal_dimension'" + \ - " with 'horizontal_loop_extent' in table {}".format(table_name)) - - # Adjust dimensions - dimensions = new_var.get_prop_value('dimensions') - if scheme_name and (table_name.endswith("_init") or table_name.endswith("_finalize")) \ - and 'horizontal_loop_extent' in dimensions: - #logging.warn("Legacy extension - replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \ - # "for variable {} in table {}".format(standard_name,table_name)) - #dimensions = ['horizontal_dimension' if x=='horizontal_loop_extent' else x for x in dimensions] - raise Exception("Legacy extension DISABLED: replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \ - "for variable {} in table {}".format(standard_name,table_name)) - elif scheme_name and table_name.endswith("_run") and 'horizontal_dimension' in dimensions: - #logging.warn("Legacy extension - replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \ - # "for variable {} in table {}".format(standard_name,table_name)) - #dimensions = ['horizontal_loop_extent' if x=='horizontal_dimension' else x for x in dimensions] - raise Exception("Legacy extension DISABLED: replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \ - "for variable {} in table {}".format(standard_name,table_name)) - elif not scheme_name and 'horizontal_dimension' in dimensions: - raise Exception("Legacy extension DISABLED: replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \ - "for variable {} in table {}".format(standard_name,table_name)) - # *DH 2020-05-26 - - if not new_var.get_prop_value('active'): - raise Exception("Unexpected result: no active attribute received from capgen metadata parser for {} / {}".format(standard_name,table_name)) - elif scheme_name and not new_var.get_prop_value('active').lower() == '.true.': - raise Exception("Scheme variable {} in table {} has metadata attribute active={}, which is not allowed".format( - standard_name, table_name, new_var.get_prop_value('active').lower())) - elif new_var.get_prop_value('active').lower() == '.true.': - active = 'T' - elif new_var.get_prop_value('active').lower() == '.false.': - active = 'F' - else: - # Replace multiple whitespaces, preserve case - active = ' '.join(new_var.get_prop_value('active').split()) - - if not new_var.get_prop_value('optional') in [False, True]: - raise Exception("Unexpected result: no optional attribute received from metadata parser for {} / {}".format(standard_name,table_name)) - elif not scheme_name and new_var.get_prop_value('optional'): - raise Exception("Host variable {} in table {} has metadata attribute optional={}, which is not allowed".format( - standard_name,table_name, new_var.get_prop_value('optional').lower())) - elif new_var.get_prop_value('optional'): - optional = 'T' - else: - optional = 'F' - - # DH* 20210812 - # Workaround for Fortran DDTs incorrectly having the type of - # the DDT copied into the kind attribute in parse_metadata_file - if new_var.is_ddt() and new_var.get_prop_value('kind'): - kind = '' - else: - kind = new_var.get_prop_value('kind') - #kind = new_var.get_prop_value('kind') - # *DH 20210812 - - # Workaround to support units with positive exponents with - # and without a plus (+) sign. Internally, we convert all - # units from capgen to the "+"-format (i.e. "m2 s-2" --> "m+2 s-2") - units = insert_plus_sign_for_positive_exponents(new_var.get_prop_value('units')) - - var = Var(standard_name = standard_name.lower(), - long_name = new_var.get_prop_value('long_name') + legacy_note, - units = units, - local_name = new_var.get_prop_value('local_name').lower(), - type = new_var.get_prop_value('type').lower(), - dimensions = [dim.lower() for dim in dimensions], - container = container, - kind = kind, - intent = new_var.get_prop_value('intent'), - active = active, - optional = optional, - ) - # Check for duplicates in same table - if standard_name in metadata.keys(): - raise Exception("Error, multiple definitions of standard name {} in new metadata table {}".format(standard_name, table_name)) - metadata[standard_name] = [var] - - return (metadata, dependencies) - -def parse_variable_tables(filepath, filename): - """Parses metadata tables on the host model side that define the available variables. - Metadata tables can refer to variables inside a module or as part of a derived - datatype, which itself is defined inside a module (depending on the location of the - metadata table). Each variable (standard_name) can exist only once, i.e. each entry - (list of variables) in the metadata dictionary contains only one element - (variable = instance of class Var defined in mkcap.py)""" - # Set debug to true if logging level is debug - debug = logging.getLogger().getEffectiveLevel() == logging.DEBUG - - # Final metadata container for all variables in file - metadata = collections.OrderedDict() - - # Registry of modules and derived data types in file - registry = collections.OrderedDict() - - # List of dependencies for this scheme - dependencies = collections.OrderedDict() - - # Read all lines of the file at once - with (open(filename, 'r')) as file: - try: - file_lines = file.readlines() - except UnicodeDecodeError: - raise Exception("Decoding error while trying to read file {}, check that the file only contains ASCII characters".format(filename)) - - lines = [] - buffer = '' - for i in range(len(file_lines)): - file_lines[i] = file_lines[i].lower() - line = file_lines[i].rstrip('\n').strip() - # Skip empty lines - if line == '' or line == '&': - continue - # Remove line continuations: concatenate with following lines - if line.endswith('&'): - buffer += file_lines[i].rstrip('\n').replace('&', ' ') - continue - # Write out line with buffer and reset buffer - lines.append(buffer + file_lines[i].rstrip('\n').replace('&', ' ')) - buffer = '' - del file_lines - - # Find all modules within the file, and save the start and end lines - module_lines = collections.OrderedDict() - line_counter = 0 - for line in lines: - words = line.split() - if len(words) > 1 and words[0].lower() in ['module', 'program'] and not words[1].lower() == 'procedure': - module_name = words[1].strip() - if module_name in registry.keys(): - raise Exception('Duplicate module name {0}'.format(module_name)) - registry[module_name] = collections.OrderedDict() - module_lines[module_name] = { 'startline' : line_counter } - elif len(words) > 1 and words[0].lower() == 'end' and words[1].lower() in ['module', 'program']: - try: - test_module_name = words[2] - except IndexError: - logging.warning('Encountered closing statement "end module" without module name; assume module_name is {0}'.format(module_name)) - test_module_name = module_name - if not module_name == test_module_name: - raise Exception('Module names in opening/closing statement do not match: {0} vs {1}'.format(module_name, test_module_name)) - module_lines[module_name]['endline'] = line_counter - line_counter += 1 - - # Parse each module in the file separately - for module_name in registry.keys(): - startline = module_lines[module_name]['startline'] - endline = module_lines[module_name]['endline'] - line_counter = 0 - in_type = False - for line in lines[startline:endline]: - # For the purpose of identifying module, type and scheme constructs, remove any trailing comments from line - if '!' in line and not line.startswith('!'): - line = line[:line.find('!')] - current_line_number = startline + line_counter - words = line.split() - for j in range(len(words)): - # Check for the word 'type', that it is the first word in the line, - # and that a name exists afterwards. It is assumed that definitions - # (not usage) of derived types cannot be nested - reasonable for Fortran. - # The following if / elif / else statements filter lines that do not - # contain a type definition. - # - # Ignore words containing type that are not 'type', 'type,', 'type::'; - # this includes variable declarations of a user defined type, e.g. 'type(mytype) ::' - if not (words[j].lower()=='type' or \ - words[j].lower().startswith('type,') or \ - words[j].lower().startswith('type::')): - continue - # Ignore variable declarations of a user defined type with a space - # between 'type' and '(', e.g. 'type (mytype) ::' - elif j == 0 and len(words) > 1 and words[j+1].startswith('('): - continue - # Ignore lines starting with 'type is' or 'type is(' (select type statements) - elif (words[j].lower() == 'type' and j==0 and j0: - continue - # Detect type definition using FtypeTypeDecl class, routine - # type_def_line and extract type_name - else: - type_declaration = FtypeTypeDecl.type_def_line(line.strip()) - if in_type: - raise Exception('Nested definitions of derived types not supported') - in_type = True - type_name = type_declaration[0] - if type_name in registry[module_name].keys(): - raise Exception('Duplicate derived type name {0} in module {1}'.format( - type_name, module_name)) - registry[module_name][type_name] = [current_line_number] - # Done with user defined type detection - line_counter += 1 - logging.debug('Parsing file {0} with registry {1}'.format(filename, registry)) - - # Variables can either be defined at module-level or in derived types - alongside with their tables - line_counter = 0 - in_table = False - in_type = False - for line in lines[startline:endline]: - current_line_number = startline + line_counter - - # Check for beginning of new table - words = line.split() - # This is case sensitive - if len(words) > 2 and words[0] in ['!!', '!>'] and '\section' in words[1] and 'arg_table_' in words[2]: - if in_table: - raise Exception('Encountered table start for table {0} while still in table {1}'.format(words[2].replace('arg_table_',''), table_name)) - table_name = words[2].replace('arg_table_','') - if not (table_name == module_name or table_name in registry[module_name].keys()): - raise Exception('Encountered table with name {0} without corresponding module or type name'.format(table_name)) - in_table = True - if not table_name in dependencies.keys(): - dependencies[table_name] = [] - header_line_number = current_line_number + 1 - line_counter += 1 - continue - # If an argument table is found, parse it - if in_table: - words = line.split('|') - # Separate the table headers - if current_line_number == header_line_number: - if 'htmlinclude' in line.lower(): - words = line.split() - if words[0] == '!!' and words[1] == '\\htmlinclude' and len(words) == 3: - filename_parts = filename.split('.') - metadata_filename = '.'.join(filename_parts[0:len(filename_parts)-1]) + '.meta' - (this_metadata, these_dependencies) = read_new_metadata(metadata_filename, module_name, table_name) - if these_dependencies: - # Remove duplicates when combining lists - dependencies[table_name] = list(set(dependencies[table_name] + these_dependencies)) - for var_name in this_metadata.keys(): - for var in this_metadata[var_name]: - if var_name in CCPP_MANDATORY_VARIABLES.keys() and not CCPP_MANDATORY_VARIABLES[var_name].compatible(var): - raise Exception('Entry for variable {0}'.format(var_name) + \ - ' in argument table {0}'.format(table_name) +\ - ' is incompatible with mandatory variable:\n' +\ - ' existing: {0}\n'.format(CCPP_MANDATORY_VARIABLES[var_name].print_debug()) +\ - ' vs. new: {0}'.format(var.print_debug())) - # Add variable to metadata dictionary - if not var_name in metadata.keys(): - metadata[var_name] = [var] - else: - for existing_var in metadata[var_name]: - if not existing_var.compatible(var): - raise Exception('New entry for variable {0}'.format(var_name) + \ - ' in argument table {0}'.format(table_name) +\ - ' is incompatible with existing entry:\n' +\ - ' existing: {0}\n'.format(existing_var.print_debug()) +\ - ' vs. new: {0}'.format(var.print_debug())) - - metadata[var_name].append(var) - else: - raise Exception("Invalid definition of new metadata format in file {}, \htmlinclude must be preceeded by '!! ' : {}".format(filename, line)) - line_counter += 1 - continue - # Check for blank table - if len(words) <= 1: - logging.debug('Skipping blank table {0}'.format(table_name)) - in_table = False - line_counter += 1 - continue - table_header = [x.strip() for x in words[1:-1]] - # Check that only valid table headers are used - for item in table_header: - if not item in VALID_ITEMS['header']: - raise Exception('Invalid column header {0} in argument table {1}'.format(item, table_name)) - # Locate mandatory column 'standard_name' - try: - standard_name_index = table_header.index('standard_name') - except ValueError: - raise Exception('Mandatory column standard_name not found in argument table {0}'.format(table_name)) - line_counter += 1 - # DH* warn or raise error for old metadata format - logging.warn("Old metadata table found for table {}".format(table_name)) - #raise Exception("Old metadata table found for table {}".format(table_name)) - # *DH - continue - else: - if len(words) == 1: - # End of table - if words[0].strip() == '!!': - if not current_line_number == header_line_number+1: - raise Exception("Invalid definition of new metadata format in file {0}".format(filename)) - in_table = False - line_counter += 1 - continue - else: - raise Exception('Encountered invalid line "{0}" in argument table {1}'.format(line, table_name)) - else: - raise Exception("Invalid definition of metadata in file {0}: {1}".format(filename, words)) - - line_counter += 1 - - # Informative output to screen - if debug and len(metadata.keys()) > 0: - for module_name in registry.keys(): - logging.debug('Module name: {0}'.format(module_name)) - container = encode_container(module_name) - vars_in_module = [] - for var_name in metadata.keys(): - for var in metadata[var_name]: - if var.container == container: - vars_in_module.append(var_name) - logging.debug('Module variables: {0}'.format(', '.join(vars_in_module))) - for type_name in registry[module_name].keys(): - container = encode_container(module_name, type_name) - vars_in_type = [] - for var_name in metadata.keys(): - for var in metadata[var_name]: - if var.container == container: - vars_in_type.append(var_name) - logging.debug('Variables in derived type {0}: {1}'.format(type_name, ', '.join(vars_in_type))) - - if len(metadata.keys()) > 0: - logging.info('Parsed variable definition tables in module {0}'.format(module_name)) - - # Add absolute path to dependencies - for table_name in dependencies.keys(): - if dependencies[table_name]: - dependencies[table_name] = [ os.path.join(filepath, x) for x in dependencies[table_name]] - for dependency in dependencies[table_name]: - if not os.path.isfile(dependency): - raise Exception("Dependency {} for variable table {} does not exit".format(dependency, table_name)) - - return (metadata, dependencies) - - -def parse_scheme_tables(filepath, filename): - """Parses metadata tables for a physics scheme that requests/requires variables as - input arguments. Metadata tables can only describe variables required by a subroutine - 'subroutine_name' of scheme 'scheme_name' inside a module 'module_name'. Each variable - (standard_name) can exist only once, i.e. each entry (list of variables) in the metadata - dictionary contains only one element (variable = instance of class Var defined in - mkcap.py). The metadata dictionaries of the individual schemes are merged afterwards - (called from ccpp_prebuild.py) using merge_metadata_dicts, where multiple instances - of variables are compared for compatibility and collected in a list (entry in the - merged metadata dictionary). The merged metadata dictionary of all schemes (which - contains only compatible variable instances in the list referred to by standard_name) - is then compared to the unique definition in the metadata dictionary of the variables - provided by the host model using compare_metadata in ccpp_prebuild.py.""" - - # Set debug to true if logging level is debug - debug = logging.getLogger().getEffectiveLevel() == logging.DEBUG - - # Final metadata container for all variables in file - metadata = collections.OrderedDict() - - # Registry of modules and derived data types in file - registry = collections.OrderedDict() - - # Argument lists of each subroutine in the file - arguments = collections.OrderedDict() - - # List of Dependencies for this scheme - dependencies = collections.OrderedDict() - - # Read all lines of the file at once - with (open(filename, 'r')) as file: - try: - file_lines = file.readlines() - except UnicodeDecodeError: - raise Exception("Decoding error while trying to read file {}, check that the file only contains ASCII characters".format(filename)) - - lines = [] - original_line_numbers = [] - buffer = '' - for i in range(len(file_lines)): - file_lines[i] = file_lines[i].lower() - line = file_lines[i].rstrip('\n').strip() - # Skip empty lines - if line == '' or line == '&': - continue - # Remove line continuations: concatenate with following lines - if line.endswith('&'): - buffer += file_lines[i].rstrip('\n').replace('&', ' ') - continue - # Write out line with buffer and reset buffer - lines.append(buffer + file_lines[i].rstrip('\n').replace('&', ' ')) - original_line_numbers.append(i+1) - buffer = '' - del file_lines - - # Find all modules within the file, and save the start and end lines - module_lines = collections.OrderedDict() - line_counter = 0 - for line in lines: - # For the purpose of identifying module constructs, remove any trailing comments from line - if '!' in line and not line.startswith('!'): - line = line[:line.find('!')] - words = line.split() - if len(words) > 1 and words[0].lower() == 'module' and not words[1].lower() == 'procedure': - module_name = words[1].strip() - if module_name in registry.keys(): - raise Exception('Duplicate module name {0}'.format(module_name)) - registry[module_name] = collections.OrderedDict() - module_lines[module_name] = { 'startline' : line_counter } - elif len(words) > 1 and words[0].lower() == 'end' and words[1].lower() == 'module': - try: - test_module_name = words[2] - except IndexError: - logging.warning('Warning, encountered closing statement "end module" without module name; assume module_name is {0}'.format(module_name)) - test_module_name = module_name - if not module_name == test_module_name: - raise Exception('Module names in opening/closing statement do not match: {0} vs {1}'.format(module_name, test_module_name)) - module_lines[module_name]['endline'] = line_counter - line_counter += 1 - - # Parse each module in the file separately - for module_name in registry.keys(): - startline = module_lines[module_name]['startline'] - endline = module_lines[module_name]['endline'] - line_counter = 0 - in_subroutine = False - for line in lines[startline:endline]: - # For the purpose of identifying scheme constructs, remove any trailing comments from line - if '!' in line and not line.startswith('!'): - line = line[:line.find('!')] - current_line_number = startline + line_counter - words = line.split() - for j in range(len(words)): - # Check for the word 'subroutine', that it is the first word in the line, - # and that a name exists afterwards. Nested subroutines are ignored. - if words[j].lower() == 'subroutine' and j == 0 and len(words) > 1: - if in_subroutine: - logging.debug('Warning, ignoring nested subroutine in module {0} and subroutine {1}'.format(module_name, subroutine_name)) - continue - subroutine_name = words[j+1].split('(')[0].strip() - # Consider the last substring separated by a '_' of the subroutine name as a 'postfix' - if subroutine_name.find('_') >= 0: - scheme_name = None - subroutine_suffix = None - for ccpp_stage in CCPP_STAGES: - pattern = '^(.*)_{}$'.format(ccpp_stage) - match = re.match(pattern, subroutine_name) - if match: - scheme_name = match.group(1) - subroutine_suffix = ccpp_stage - break - if match: - if not scheme_name == module_name: - raise Exception('Scheme name differs from module name: module_name="{0}" vs. scheme_name="{1}"'.format( - module_name, scheme_name)) - if not scheme_name in registry[module_name].keys(): - registry[module_name][scheme_name] = collections.OrderedDict() - if subroutine_name in registry[module_name][scheme_name].keys(): - raise Exception('Duplicate subroutine name {0} in module {1}'.format( - subroutine_name, module_name)) - registry[module_name][scheme_name][subroutine_name] = [current_line_number] - in_subroutine = True - elif words[j].lower() == 'subroutine' and j == 1 and words[j-1].lower() == 'end': - try: - test_subroutine_name = words[j+1] - except IndexError: - logging.warning('Warning, encountered closing statement "end subroutine" without subroutine name; ' +\ - ' assume subroutine_name is {0}'.format(subroutine_name)) - test_subroutine_name = subroutine_name - if in_subroutine and subroutine_name == test_subroutine_name: - in_subroutine = False - registry[module_name][scheme_name][subroutine_name].append(current_line_number) - # Avoid problems by enforcing end statements to carry a descriptor (subroutine, module, ...) - elif in_subroutine and len(words) == 1 and words[0].lower() == 'end': - raise Exception('Encountered closing statement "end" without descriptor (subroutine, module, ...): ' +\ - 'line {0}="{1}" in file {2}'.format(original_line_numbers[current_line_number], line, filename)) - line_counter += 1 - - # Check that for each registered subroutine the start and end lines were found - for scheme_name in registry[module_name].keys(): - for subroutine_name in registry[module_name][scheme_name].keys(): - if not len(registry[module_name][scheme_name][subroutine_name]) == 2: - raise Exception('Error parsing start and end lines for subroutine {0} in module {1}'.format(subroutine_name, module_name)) - logging.debug('Parsing file {0} with registry {1}'.format(filename, registry)) - - for scheme_name in registry[module_name].keys(): - # Record the dependencies for the scheme - if not scheme_name in dependencies.keys(): - dependencies[scheme_name] = [] - for subroutine_name in registry[module_name][scheme_name].keys(): - # Record the order of variables in the call list to each subroutine in a list - if not scheme_name in arguments.keys(): - arguments[scheme_name] = collections.OrderedDict() - if not subroutine_name in arguments[scheme_name].keys(): - arguments[scheme_name][subroutine_name] = [] - # Find the argument table corresponding to each subroutine by searching - # "upward" from the subroutine definition line for the "arg_table_SubroutineName" section - table_found = False - header_line_number = None - for line_number in range(registry[module_name][scheme_name][subroutine_name][0], -1, -1): - line = lines[line_number] - words = line.split() - for word in words: - if (len(words) > 2 and words[0] in ['!!', '!>'] and '\section' in words[1] and 'arg_table_{0}'.format(subroutine_name) in words[2]): - table_found = True - header_line_number = line_number + 1 - table_name = subroutine_name - break - else: - for word in words: - if 'arg_table_{0}'.format(subroutine_name) in word: - raise Exception("Malformatted table found in {0} / {1} / {2} / {3}".format(filename, module_name, scheme_name, subroutine_name)) - if table_found: - break - # If an argument table is found, parse it - if table_found: - if 'htmlinclude' in lines[header_line_number].lower(): - words = lines[header_line_number].split() - if words[0] == '!!' and words[1] == '\\htmlinclude' and len(words) == 3: - filename_parts = filename.split('.') - metadata_filename = '.'.join(filename_parts[0:len(filename_parts)-1]) + '.meta' - (this_metadata, these_dependencies) = read_new_metadata(metadata_filename, module_name, table_name, - scheme_name=scheme_name, subroutine_name=subroutine_name) - if these_dependencies: - # Remove duplicates when combining lists - dependencies[scheme_name] = list(set(dependencies[scheme_name] + these_dependencies)) - for var_name in this_metadata.keys(): - # Add standard_name to argument list for this subroutine - arguments[scheme_name][subroutine_name].append(var_name) - # For all instances of this var (can be only one) in this subroutine's metadata, - # add to global metadata and check for compatibility with existing variables - for var in this_metadata[var_name]: - if not var_name in metadata.keys(): - metadata[var_name] = [var] - else: - for existing_var in metadata[var_name]: - if not existing_var.compatible(var): - raise Exception('New entry for variable {0}'.format(var_name) + \ - ' in argument table of subroutine {0}'.format(subroutine_name) +\ - ' is incompatible with existing entry:\n' +\ - ' existing: {0}\n'.format(existing_var.print_debug()) +\ - ' vs. new: {0}'.format(var.print_debug())) - metadata[var_name].append(var) - else: - raise Exception("Invalid definition of new metadata format in file {}, \htmlinclude must be preceeded by '!! ' : {}".format(filename, lines[header_line_number])) - # Next line must denote the end of table, - # i.e. look for a line containing only '!!' - line_number = header_line_number+1 - nextline = lines[line_number] - nextwords = nextline.split() - if len(nextwords) == 1 and nextwords[0].strip() == '!!': - end_of_table = True - else: - raise Exception('Encountered invalid format "{0}" of new metadata table hook in table {1}'.format(line, table_name)) - line_number += 1 - - else: - words = lines[header_line_number].split() - if len(words) == 1 and words[0].strip() == '!!': - logging.info("Legacy extension - skip empty table for {}".format(table_name)) - end_of_table = True - line_number += 1 - else: - raise Exception("Invalid definition of metadata in file {0}: {1}".format(filename, words)) - - # After parsing entire metadata table for the subroutine, check that all - # mandatory CCPP variables are present - skip empty tables. - if arguments[scheme_name][subroutine_name]: - for var_name in CCPP_MANDATORY_VARIABLES.keys(): - if not var_name in arguments[scheme_name][subroutine_name]: - raise Exception('Mandatory CCPP variable {0} not declared in metadata table of subroutine {1}'.format( - var_name, subroutine_name)) - # Sort the dependencies to avoid differences in the auto-generated code - dependencies[scheme_name].sort() - - # Debugging output to screen and to XML - if debug and len(metadata.keys()) > 0: - # To screen - logging.debug('Module name: {}'.format(module_name)) - for scheme_name in registry[module_name].keys(): - logging.debug('Scheme name: {}'.format(scheme_name)) - logging.debug('Scheme dependencies: {}'.format(dependencies[scheme_name])) - for subroutine_name in registry[module_name][scheme_name].keys(): - container = encode_container(module_name, scheme_name, subroutine_name) - vars_in_subroutine = [] - for var_name in metadata.keys(): - for var in metadata[var_name]: - if var.container == container: - vars_in_subroutine.append(var_name) - logging.debug('Variables in subroutine {}: {}'.format(subroutine_name, ', '.join(vars_in_subroutine))) - # Standard output to screen - elif len(metadata.keys()) > 0: - for scheme_name in registry[module_name].keys(): - if dependencies[scheme_name]: - logging.info('Parsed tables in scheme {} with dependencies {}'.format(scheme_name, dependencies[scheme_name])) - else: - logging.info('Parsed tables in scheme {}'.format(scheme_name)) - - # End of loop over all module_names - - # Add absolute path to dependencies - for scheme_name in dependencies.keys(): - if dependencies[scheme_name]: - dependencies[scheme_name] = [ os.path.join(filepath, x) for x in dependencies[scheme_name]] - for dependency in dependencies[scheme_name]: - if not os.path.isfile(dependency): - raise Exception("Dependency {} for scheme table {} does not exit".format(dependency, scheme_name)) - - return (metadata, arguments, dependencies) diff --git a/scripts/metadata_table.py b/scripts/metadata_table.py deleted file mode 100755 index b657dd2c..00000000 --- a/scripts/metadata_table.py +++ /dev/null @@ -1,1504 +0,0 @@ -#!/usr/bin/env python3 -""" -There are four types of CCPP metadata tables, scheme, module, ddt, and host. -A metadata file contains one or more metadata tables. -A metadata file SHOULD NOT mix metadata table types. The exception is a - metadata file which contains one or more ddt tables followed by a module - or host table. - -Each metadata table begins with a 'ccpp-table-properties' section followed by - one or more 'ccpp-arg-table' sections. These sections are described below. -A 'ccpp-arg-table' section is followed by one or more variable declaration - sections, also described below. - -Metadata headers are in config file format. - -A 'ccpp-table-properties' section entries are: -name = : the name of the following ccpp-arg-table entries (required). - It is one of the following possibilities: - - SchemeName: the name of a scheme (i.e., the name of - a scheme interface (related to SubroutineName below). - - DerivedTypeName: a derived type name for a type which will be used - somewhere in the CCPP interface. - - ModuleName: the name of the module whose module variables will be - used somewhere in the CCPP interface - - HostName: the name of the host model. Variables in this section become - part of the CCPP UI, the CCPP routines called by the - host model (e.g., _ccpp_physics_run). -type = : The type of header (required), one of: - - scheme: A CCPP subroutine - - ddt: A header for a derived data type - - module: A header on some module data - - host: A header on data which will be part of the CCPP UI -dependencies = : Comma-separated list of module dependencies - Each item should appear in one or more use statements in the - corresponding Fortran module -dependencies_path = : A path, relative to the location of - metadata table file, where dependencies can be found. -module_name = : only needed if module name differs from filename -source_path = -dynamic_constituent_routine = ??? : @peverwhee? -kind_spec = : One or more optional Fortran kinds defined - in the corresponding Fortran file. - The format is fortran_module:ccpp_kind_name=>kind_name or - fortran_module:kind_name - - is the module name of the corresponding Fortran module - - is defined in the corresponding Fortran module - - is optional and describes the kind name used in CCPP - metadata tables and Fortran files. - These entries are added to the framework_env object and - thus to ccpp_kinds.F90 - The entries in ccpp_kinds.F90 are: - use , only - use , only => - where the first form is used if is omitted. - -The ccpp-arg-table section entries in this section are: -name = : the name of the file object which immediately follows the - argument table (required). - It is one of the following possibilities: - - SubroutineName: the name of a subroutine (i.e., the name of - a scheme interface function such as SchemeName_run) - - DerivedTypeName: a derived type name for a type which will be used - somewhere in the CCPP interface. - - ModuleName: the name of the module whose module variables will be - used somewhere in the CCPP interface - - HostName: the name of the host model. Variables in this section become - part of the CCPP UI, the CCPP routines called by the - host model (e.g., _ccpp_physics_run). -type = : The type of header (required). It must match the type of the - associated ccpp-table-properties section (see above). - -A variable declaration section begins with a variable name line (a local -variable name enclosed in square brackets) followed by one or more -variable attribute statements. -A variable attribute statement is an attribute name and the value for -that attribute separated by an equal sign. Whitespace is not -significant except inside of strings. -Variable attribute statements may be combined on a line if separated by -a vertical bar. - -An example argument table is shown below. - -[ccpp-table-properties] - name = - type = scheme - dependencies_path = - dependencies = - module_name = # only needed if module name differs from filename - source_path = - dynamic_constituent_routine = - -[ccpp-arg-table] - name = - type = scheme -[ im ] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent, start at 1 - units = index - type = integer - dimensions = () - intent = in -[ ix ] - standard_name = horizontal_loop_dimension - long_name = horizontal dimension - units = index | type = integer | dimensions = () - intent = in - ... -[ errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - type = character - len = * - dimensions = () - intent = out -[ ierr ] - standard_name = ccpp_error_code - long_name = error flag for error handling in CCPP - type = integer - units = 1 - dimensions = () - intent=out - -Notes on the input format: -- SubroutineName must match the name of the subroutine that the argument - table describes -- DerivedTypeName must match the name of the derived type that the argument - table describes -- ModuleName must match the name of the module whose variables the argument - table describes -- for variable type definitions and module variables, the intent keyword - is not functional and should be omitted -- each argument table (and its subroutine) must accept the following two arguments for error handling (the local name can vary): -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -""" - -# Python library imports -import difflib -import logging -import os.path -import re -# CCPP framework imports -from ccpp_state_machine import CCPP_STATE_MACH -from metavar import Var, VarDictionary, CCPP_CONSTANT_VARS -from parse_tools import ParseObject, ParseSource, ParseContext, context_string -from parse_tools import ParseInternalError, ParseSyntaxError, CCPPError -from parse_tools import FORTRAN_ID, FORTRAN_SCALAR_REF, FORTRAN_SCALAR_REF_RE -from parse_tools import check_fortran_ref, check_fortran_id -from parse_tools import check_fortran_intrinsic -from parse_tools import register_fortran_ddt_name, unique_standard_name - -######################################################################## - -SCHEME_HEADER_TYPE = 'scheme' -_SINGLETON_TABLE_TYPES = ['ddt', 'host', 'module'] # Only one section per table -TABLE_TYPES = _SINGLETON_TABLE_TYPES + [SCHEME_HEADER_TYPE] -HEADER_TYPES = TABLE_TYPES + ['local'] -UNKNOWN_PROCESS_TYPE = 'UNKNOWN' - -_BLANK_LINE = re.compile(r"\s*[#;]") - -def blank_metadata_line(line): - """Return True if is a valid config format blank or comment - line. Also return True if we have reached the end of the file - (no line)""" - return (not line) or (_BLANK_LINE.match(line) is not None) - -######################################################################## - -def _parse_config_line(line, context): - """Parse a config line and return a list of keyword value pairs.""" - parse_items = [] - if line is None: - pass # No properties on this line - elif blank_metadata_line(line): - pass # No properties on this line - else: - properties = line.strip().split('|') - for prop in properties: - pitems = [x.strip() for x in prop.split('=', 1)] - if len(pitems) >= 2: - parse_items.append(pitems) - else: - raise ParseSyntaxError("variable property syntax", - token=prop, context=context) - # end if - # end for - # end if - return parse_items - -######################################################################## - -def parse_metadata_file(filename, known_ddts, run_env, skip_ddt_check=False, relative_source_path=False): - """Parse and return list of parsed metadata tables""" - # Read all lines of the file at once - meta_tables = [] - table_titles = [] # Keep track of names in file - with open(filename, 'r') as infile: - fin_lines = infile.readlines() - for index, fin_line in enumerate(fin_lines): - fin_lines[index] = fin_line.rstrip('\n') - # end for - # end with - # Look for a header start - parse_obj = ParseObject(filename, fin_lines) - curr_line, curr_line_num = parse_obj.curr_line() - while curr_line is not None: - if MetadataTable.table_start(curr_line): - new_table = MetadataTable(run_env, parse_object=parse_obj, - known_ddts=known_ddts, - skip_ddt_check=skip_ddt_check, - relative_source_path=relative_source_path) - ntitle = new_table.table_name - if ntitle not in table_titles: - meta_tables.append(new_table) - table_titles.append(ntitle) - if new_table.table_type == 'ddt': - known_ddts.append(ntitle.lower()) - # end if - else: - errmsg = 'Duplicate metadata table, {}, at {}:{}' - ctx = curr_line_num + 1 - raise CCPPError(errmsg.format(ntitle, filename, ctx)) - # end if - curr_line, curr_line_num = parse_obj.curr_line() - elif blank_metadata_line(curr_line): - curr_line, curr_line_num = parse_obj.next_line() - else: - raise ParseSyntaxError('CCPP metadata line', token=curr_line, - context=parse_obj) - # end if - # end while - return meta_tables - -######################################################################## - -def find_scheme_names(filename): - """Find and return a list of all the physics scheme names in - . A scheme is identified by its ccpp-table-properties name. - """ - scheme_names = [] - with open(filename, 'r') as infile: - fin_lines = infile.readlines() - # end with - num_lines = len(fin_lines) - context = ParseContext(linenum=1, filename=filename) - while context.line_num <= num_lines: - if MetadataTable.table_start(fin_lines[context.line_num - 1]): - found_start = False - while not found_start: - line = fin_lines[context.line_num].strip() - context.line_num += 1 - if line and (line[0] == '['): - found_start = True - elif line: - props = _parse_config_line(line, context) - for prop in props: - # Look for name property - key = prop[0].lower() - value = prop[1] - if key == 'name': - scheme_names.append(value) - # end if - # end for - # end if - if context.line_num > num_lines: - break - # end if - # end while - else: - context.line_num += 1 - # end if - # end while - return scheme_names - -######################################################################## - -def register_ddts(file_list): - """Scan the metadata files in and register all - DDT tables found. - Return a list of the DDTs type names found. - """ - errors = "" - ddt_names = set() - for mfile in file_list: - if os.path.exists(mfile): - with open(mfile, 'r') as infile: - fin_lines = infile.readlines() - # end with - pobj = ParseObject(mfile, fin_lines) - in_table = False # Line number of table start - ddt_name = "" - table_is_ddt = False - # Search the file for ccpp-table-properties sections - curr_line, line_num = pobj.next_line() - while(curr_line is not None): - if in_table: - # We are in a table properties sec, look for name and type - if MetadataSection.header_start(curr_line) or \ - MetadataTable.table_start(curr_line): - # We have exited the table, record if a DDT - if table_is_ddt: - if ddt_name: - ddt_names.add(ddt_name) - else: - emsg = "Unnamed CCPP metadata table" - pobj.add_syntax_err(emsg) - # end if - # end if - if MetadataTable.table_start(curr_line): - in_table = line_num + 1 - else: - in_table = False - # end if - ddt_name = "" - table_is_ddt = False - else: - for prop in _parse_config_line(curr_line, context=pobj): - if prop[0].lower() == 'name': - ddt_name = prop[1].lower() - elif prop[0].lower() == 'type': - table_is_ddt = prop[1].lower() == 'ddt' - # end if - # end for - # end if - elif MetadataTable.table_start(curr_line): - in_table = line_num + 1 - # end if - curr_line, line_num = pobj.next_line() - # end while - if pobj.error_message: - if errors: - errors += "\n" - # end if - errors += pobj.error_message - # end if - else: - if errors: - errors += "\n" - # end if - errors += f"Metadata file, '{mfile}', not found." - # end if - # end for - if in_table: - # This is a malformed CCPP metadata file! - if errors: - errors += "\n" - # end if - errors += f"Malformed CCPP metadata file, '{mfile}'" - # end if - if errors: - raise CCPPError(f"{errors}") - else: - for ddt in ddt_names: - register_fortran_ddt_name(ddt) - # end for - # end if - return list(ddt_names) - -######################################################################## - -class MetadataTable(): - """Class to hold a CCPP Metadata table including the table header - (ccpp-table-properties section) and all of the associated table - sections (ccpp-arg-table sections).""" - - __table_start = re.compile(r"(?i)\s*\[\s*ccpp-table-properties\s*\]") - - def __init__(self, run_env, table_name_in=None, table_type_in=None, - dependencies=None, dependencies_path=None, source_path=None, - known_ddts=None, var_dict=None, module=None, parse_object=None, - skip_ddt_check=False, relative_source_path=False): - """Initialize a MetadataTable, either with a name, , and - type, , or with information from a file (). - if is None, , , and - are are also stored. - If and / or module are passed (not allowed with - ', maxsplit=1)] - if len(spec_list) > 1: - new_kind = spec_list[1] - new_ccpp_kind = spec_list[0] - else: - new_kind = new_ccpp_kind - # end if - try: - check_fortran_id(new_kind, {}, True) - except CCPPError as err: - self.__pobj.add_syntax_err(f"{err}") - new_kind = None - # end try - if new_kind and (new_ccpp_kind != new_kind): - try: - check_fortran_id(new_ccpp_kind, {}, True) - except CCPPError as err: - self.__pobj.add_syntax_err(f"{err}") - new_ccpp_kind = None - # end try - # end if - if new_kind: - emsg = run_env.add_kind_type(new_ccpp_kind, - new_kind, fort_module) - if emsg: - self.__pobj.add_syntax_err(emsg) - # end if - # end if - else: - tok_type = "metadata table start property" - self.__pobj.add_syntax_err(tok_type, token=key) - # end if - # end for - curr_line, _ = self.__pobj.next_line() - else: - # Process a metadata section - if MetadataSection.header_start(curr_line): - skip_rest_of_section = False - section = MetadataSection(self.table_name, self.table_type, - run_env, parse_object=self.__pobj, - module=self.__module_name, - known_ddts=known_ddts, - skip_ddt_check=skip_ddt_check) - # Some table types only allow for one associated section - if ((len(self.__sections) == 1) and - (self.table_type in _SINGLETON_TABLE_TYPES)): - prev_title = self.__sections[0].title - emsg = "{}, '{}', table already contains '{}'" - self.__pobj.add_syntax_err(emsg.format(self.table_type, - section.title, - prev_title)) - # end if - self.__sections.append(section) - # Note: Do not read next line, we are already on it. - curr_line, _ = self.__pobj.curr_line() - elif not blank_metadata_line(curr_line): - if not skip_rest_of_section: - self.__pobj.add_syntax_err("metadata file line", - token=curr_line) - skip_rest_of_section = True - # end if - curr_line, _ = self.__pobj.next_line() - else: - curr_line, _ = self.__pobj.next_line() - # end if - # end if - # end while - if self.__pobj.error_message: - # Time to dump out error messages - raise CCPPError(self.__pobj.error_message) - # end if - if self.table_type == "ddt": - known_ddts.append(self.table_name.lower()) - # end if - if self.__dependencies is None: - self.__dependencies = [] - # end if - - def start_context(self, with_comma=True, nodir=True): - """Return a context string for the beginning of the table""" - return context_string(self.__start_context, - with_comma=with_comma, nodir=nodir) - - def sections(self): - """Return the metadata header sections for this table""" - if self.__sections: - # Return a copy so it cannot be modified - return list(self.__sections) - return self.__sections - - @property - def table_name(self): - 'Return the name of the metadata table' - return self.__table_name - - @property - def table_type(self): - 'Return the type of structure this header documents' - return self.__table_type - - @property - def dependencies(self): - """Return the dependencies for this table""" - return self.__dependencies - - @property - def module_name(self): - """Return the module name for this metadata table""" - return self.__module_name - - @property - def dependencies_path(self): - """Return the relative path for the table's dependencies""" - return self.__dependencies_path - - @property - def fortran_source_path(self): - """Return the Fortran source path for this table""" - return self.__fortran_src_path - - @property - def run_env(self): - """Return this table's CCPPFrameworkEnv object""" - return self.__run_env - - def __repr__(self): - '''Print representation for MetadataTable objects''' - return "<{} {} @ 0X{:X}>".format(self.__class__.__name__, - self.table_name, id(self)) - - def __str__(self): - '''Print string for MetadataTable objects''' - return "<{} {}>".format(self.__class__.__name__, self.table_name) - - @classmethod - def table_start(cls, line): - """Return True iff is a ccpp-table-properties header statement. - """ - if (line is None) or blank_metadata_line(line): - match = None - else: - match = cls.__table_start.match(line) - # end if - return match is not None - -######################################################################## - -class MetadataSection(ParseSource): - """Class to hold all information from a metadata header - >>> from framework_env import CCPPFrameworkEnv - >>> _DUMMY_RUN_ENV = CCPPFrameworkEnv(None, {'host_files':'', \ - 'scheme_files':'', \ - 'suites':''}) - >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV, module="foo", \ - parse_object=ParseObject("foobar.txt", \ - ["name = footable", "type = scheme", \ - "[ im ]", "standard_name = horizontal_loop_extent", \ - "long_name = horizontal loop extent, start at 1", \ - "units = index | type = integer", \ - "dimensions = () | intent = in"])) #doctest: +ELLIPSIS - - >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV, module="foobar", \ - parse_object=ParseObject("foobar.txt", \ - ["name = footable", "type = scheme", \ - "[ im ]", "standard_name = horizontal_loop_extent", \ - "long_name = horizontal loop extent, start at 1", \ - "units = index | type = integer", \ - "dimensions = () | intent = in"])).find_variable('horizontal_loop_extent') #doctest: +ELLIPSIS - - >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV, module="foobar", \ - parse_object=ParseObject("foobar.txt", \ - ["name = footable", "type = scheme", \ - "process = microphysics", "[ im ]", \ - "standard_name = horizontal_loop_extent", \ - "long_name = horizontal loop extent, start at 1", \ - "units = index | type = integer", \ - "dimensions = () | intent = in"])).find_variable('horizontal_loop_extent') #doctest: +ELLIPSIS - - >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV, module="foo", \ - parse_object=ParseObject("foobar.txt", \ - ["name = footable", "type=scheme", \ - "[ im ]", "standard_name = horizontal_loop_extent", \ - "long_name = horizontal loop extent, start at 1", \ - "units = index | type = integer", \ - "dimensions = () | intent = in", \ - " subroutine foo()"])).find_variable('horizontal_loop_extent') #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - parse_source.ParseSyntaxError: Invalid variable property syntax, 'subroutine foo()', at foobar.txt:9 - >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV, module="foobar", \ - parse_object=ParseObject("foobar.txt", \ - ["name = footable", "type = scheme", \ - "[ im ]", "standard_name = horizontal_loop_extent", \ - "long_name = horizontal loop extent, start at 1", \ - "units = index | type = integer", \ - "dimensions = () | intent = in", \ - ""], line_start=0)).find_variable('horizontal_loop_extent').get_prop_value('local_name') - 'im' - >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV, module="foo", \ - parse_object=ParseObject("foobar.txt", \ - ["name = footable", "type = scheme" \ - "[ im ]", "standard_name = horizontalloop extent", \ - "long_name = horizontal loop extent, start at 1", \ - "units = index | type = integer", \ - "dimensions = () | intent = in", \ - ""], line_start=0)).find_variable('horizontal_loop_extent') - - >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV, module="foo", \ - parse_object=ParseObject("foobar.txt", \ - ["[ccpp-arg-table]", "name = foobar", "type = scheme" \ - "[ im ]", "standard_name = horizontal loop extent", \ - "long_name = horizontal loop extent, start at 1", \ - "units = index | type = integer", \ - "dimensions = () | intent = in", \ - ""], line_start=0)).find_variable('horizontal_loop_extent') - - >>> MetadataSection("foobar", "scheme", _DUMMY_RUN_ENV, module="foo", \ - parse_object=ParseObject("foobar.txt", \ - ["name = foobar" \ - "[ im ]", "standard_name = horizontal loop extent", \ - "long_name = horizontal loop extent, start at 1", \ - "units = index | type = integer", \ - "dimensions = () | intent = in", \ - ""], line_start=0)).find_variable('horizontal_loop_extent') - - >>> MetadataSection("foobar", "scheme", _DUMMY_RUN_ENV, module="foo", \ - parse_object=ParseObject("foobar.txt", \ - ["name = foobar", "foo = bar" \ - "[ im ]", "standard_name = horizontal loop extent", \ - "long_name = horizontal loop extent, start at 1", \ - "units = index | type = integer", \ - "dimensions = () | intent = in", \ - ""], line_start=0)).find_variable('horizontal_loop_extent') - - >>> MetadataSection.header_start('[ ccpp-arg-table ]') - True - >>> MetadataSection.header_start('[ qval ]') - False - >>> MetadataSection.header_start(' local_name = foo') - False - >>> MetadataSection.variable_start('[ qval ]', ParseObject('foo.meta', [])) - 'qval' - >>> MetadataSection.variable_start('[ qval(hi_mom) ]', ParseObject('foo.meta', [])) - 'qval(hi_mom)' - >>> MetadataSection.variable_start(' local_name = foo', ParseContext(filename='foo.meta', linenum=1)) - -""" - - __header_start = re.compile(r"(?i)\s*\[\s*ccpp-arg-table\s*\]") - - __var_start = re.compile(r"^\[\s*"+FORTRAN_ID+r"\s*\]$") - - __vref_start = re.compile(r"^\[\s*"+FORTRAN_SCALAR_REF+r"\s*\]$") - - __html_template__ = """ - - -{title} - - - -
-{header}{contents}
- - -""" - - def __init__(self, table_name, table_type, run_env, parse_object=None, - title=None, type_in=None, module=None, process_type=None, - var_dict=None, known_ddts=None, skip_ddt_check=False): - """Initialize a new MetadataSection object. - If is not None, initialize from the current file and - location in . - If is None, initialize from , <type>, <module>, - and <var_dict>. Note that if <parse_object> is not None, <title>, - <type>, <module>, and <var_dict> are ignored. - <table_name> and <table_type> are the name and type of the - metadata header of which this section is a part. They must match - the type and name of this section (once the name action has been - removed, e.g., name = foo_init matches type foo). - """ - self.__pobj = parse_object - self.__variables = None # In case __init__ crashes - self.__section_title = None - self.__header_type = None - self.__module_name = module - self.__process_type = UNKNOWN_PROCESS_TYPE - self.__section_valid = True - self.__run_env = run_env - if parse_object is None: - if title is not None: - self.__section_title = title - else: - raise ParseInternalError('MetadataSection requires a title') - # end if - if type_in is None: - perr = 'MetadataSection requires a header type' - raise ParseInternalError(perr) - # end if - if type_in in HEADER_TYPES: - self.__header_type = type_in - else: - self.__pobj.add_syntax_err("metadata arg table type", - token=type_in) - self.__section_valid = False - # end if - mismatch = self.section_table_mismatch(table_name, table_type) - if mismatch: - self.__pobj.add_syntax_err(mismatch) - self.__section_valid = False - # end if - mismatch = self.section_table_mismatch(table_name, table_type) - if mismatch: - raise CCPPError(mismatch) - # end if - if module is None: - perr = "MetadataSection requires a module name" - self.__pobj.add_syntax_err(perr) - self.__section_valid = False - # end if - if process_type is None: - self.__process_type = UNKNOWN_PROCESS_TYPE - else: - self.__process_type = process_type - # end if - # Initialize our ParseSource parent - super().__init__(self.title, self.header_type, self.__pobj) - self.__variables = VarDictionary(self.title, run_env) - for var in var_dict.variable_list(): # Let this crash if no dict - self.__variables.add_variable(var, run_env) - # end for - self.__start_context = None - else: - if known_ddts is None: - known_ddts = [] - # end if - self.__start_context = ParseContext(context=self.__pobj) - self.__init_from_file(table_name, table_type, known_ddts, self.module, - run_env, skip_ddt_check=skip_ddt_check) - # end if - # Register this header if it is a DDT - if self.header_type == 'ddt': - register_fortran_ddt_name(self.title) - # end if - # Categorize the variables - self._var_intents = {'in' : [], 'out' : [], 'inout' : []} - for var in self.variable_list(): - intent = var.get_prop_value('intent') - if intent is not None: - self._var_intents[intent].append(var) - # end if - # end for - - def __init_from_file(self, table_name, table_type, known_ddts, module_name, - run_env, skip_ddt_check=False): - """ Read the section preamble, assume the caller already figured out - the first line of the header using the header_start method.""" - start_ctx = context_string(self.__pobj) - curr_line, _ = self.__pobj.next_line() # Skip past [ccpp-arg-table] - self.__module_name = module_name - while ((curr_line is not None) and - (not MetadataSection.variable_start(curr_line, self.__pobj)) and - (not MetadataSection.header_start(curr_line)) and - (not MetadataTable.table_start(curr_line))): - for prop in _parse_config_line(curr_line, self.__pobj): - # Manually parse name, type, and module properties - key = prop[0].lower() - value = prop[1] - if key == 'name': - self.__section_title = value - elif key == 'type': - if value not in HEADER_TYPES: - self.__pobj.add_syntax_err("metadata table type", - token=value) - self.__section_valid = False - close = difflib.get_close_matches(value, HEADER_TYPES) - if close: - self.__header_type = close[0] # Allow error continue - # end if - # end if - # Set value even if error so future error msgs make sense - self.__header_type = value - elif key == 'process': - self.__process_type = value - else: - self.__pobj.add_syntax_err("metadata table start property", - token=key) - self.__process_type = 'INVALID' # Allow error continue - self.__section_valid = False - # end if - # end for - curr_line, _ = self.__pobj.next_line() - # end while - if self.title is None: - self.__pobj.add_syntax_err("metadata header start, no table name", - token=curr_line) - self.__section_valid = False - # end if - if self.header_type is None: - self.__pobj.add_syntax_err("metadata header start, no table type", - token=curr_line) - self.__section_valid = False - # end if - if ((self.header_type != SCHEME_HEADER_TYPE) and - (self.process_type != UNKNOWN_PROCESS_TYPE)): - emsg = "process keyword only allowed for a scheme" - self.__pobj.add_syntax_err(emsg, token=curr_line) - self.__process_type = UNKNOWN_PROCESS_TYPE # Allow error continue - self.__section_valid = False - # end if - mismatch = self.section_table_mismatch(table_name, table_type) - if mismatch: - self.__pobj.add_syntax_err(mismatch) - self.__section_valid = False - # end if - if run_env.verbose: - run_env.logger.info("Parsing {} {}{}".format(self.header_type, - self.title, start_ctx)) - # end if - if self.header_type == "ddt": - known_ddts.append(self.title.lower()) - # end if - # Initialize our ParseSource parent - super().__init__(self.title, self.header_type, self.__pobj) - # Read the variables - valid_lines = True - self.__variables = VarDictionary(self.title, run_env) - while valid_lines: - newvar, curr_line = self.parse_variable(curr_line, known_ddts, - skip_ddt_check=skip_ddt_check) - valid_lines = newvar is not None - if valid_lines: - if run_env.verbose: - dmsg = 'Adding {} to {}' - lname = newvar.get_prop_value('local_name') - run_env.logger.debug(dmsg.format(lname, self.title)) - # end if - self.__variables.add_variable(newvar, run_env) - # Check to see if we hit the end of the table - valid_lines = not MetadataSection.header_start(curr_line) - else: - # We have a bad variable, see if we have more variables - lname = MetadataSection.variable_start(curr_line, self.__pobj) - valid_lines = lname is not None - # end while - # end if - # end while - - def parse_variable(self, curr_line, known_ddts, skip_ddt_check=False): - """Parse a new metadata variable beginning on <curr_line>. - The header line has the format [ <valid_fortran_symbol> ]. - """ - newvar = None - var_ok = True # Set to False if an error is detected - valid_line = ((curr_line is not None) and - (not MetadataSection.header_start(curr_line)) and - (not MetadataTable.table_start(curr_line))) - if valid_line: - # variable_start handles exception - local_name = MetadataSection.variable_start(curr_line, self.__pobj).lower() - else: - local_name = None - # end if - if local_name is None: - # This is not a valid variable line, punt (should be end of table) - valid_line = False - # end if - # Parse lines until invalid line is found - # NB: Header variables cannot have embedded blank lines - if valid_line: - var_props = {} - var_props['local_name'] = local_name - # Grab context that points at beginning of definition - context = ParseContext(context=self.__pobj) - else: - var_props = None - # end if - while valid_line: - curr_line, _ = self.__pobj.next_line() - valid_line = ((curr_line is not None) and - (not MetadataSection.header_start(curr_line)) and - (not MetadataTable.table_start(curr_line)) and - (MetadataSection.variable_start(curr_line, - self.__pobj) is None)) - # A valid line may have multiple properties (separated by '|') - if valid_line: - properties = _parse_config_line(curr_line, self.__pobj) - for prop in properties: - pname = prop[0].lower() - pval_str = prop[1] - if ((pname == 'type') and - (not check_fortran_intrinsic(pval_str, error=False))): - if skip_ddt_check or pval_str.lower() in known_ddts: - if skip_ddt_check: - register_fortran_ddt_name(pval_str) - # end if - pval = pval_str.lower() - pname = 'ddt_type' - else: - errmsg = "Unknown DDT type, {}".format(pval_str) - self.__pobj.add_syntax_err(errmsg) - self.__section_valid = False - var_ok = False - # end if - else: - # Make sure this is a match - check_prop = Var.get_prop(pname) - if check_prop is not None: - pval = check_prop.valid_value(pval_str) - else: - emsg = "variable property name" - self.__pobj.add_syntax_err(emsg, token=pname) - self.__section_valid = False - var_ok = False - # end if - if pval is None: - errmsg = "'{}' property value" - self.__pobj.add_syntax_err(errmsg.format(pname), - token=pval_str) - self.__section_valid = False - var_ok = False - # end if - # end if - if var_ok: - # If we get this far, we have a valid property. - # Special case for dimensions, turn them into ranges - if pname == 'dimensions': - porig = pval - pval = [] - for dim in porig: - if ':' in dim: - for dim2 in dim.split(':'): - dim_ok = VarDictionary.loop_var_okay(standard_name=dim2, - is_run_phase=self.__section_title.endswith("_run")) - if not dim_ok: - emsg = "horizontal dimension" - self.__pobj.add_syntax_err(emsg, token=dim2) - self.__section_valid = False - var_ok = False - # end if - # end for - pval.append(dim.lower()) - else: - dim_ok = VarDictionary.loop_var_okay(standard_name=dim, - is_run_phase=self.__section_title.endswith("_run")) - if not dim_ok: - emsg = "horizontal dimension" - self.__pobj.add_syntax_err(emsg, token=dim) - self.__section_valid = False - var_ok = False - # end if - cone_str = 'ccpp_constant_one:{}' - pval.append(cone_str.format(dim.lower())) - # end if - # end for - # end if - # Special handling for standard_names (convert to lowercase) - if pname == 'standard_name': - pval = pval.lower() - # end if - # Add the property to our Var dictionary - var_props[pname] = pval - # end if - # end for - # end if - # end while - if var_ok and (var_props is not None): - # Check for array reference - sub_name = MetadataSection.check_array_reference(local_name, - var_props, context) - if sub_name: - var_props['local_name'] = sub_name - # end if (else just leave the local name alone) - try: - newvar = Var(var_props, self, self.run_env, context=context) - except CCPPError as verr: - self.__pobj.add_syntax_err(verr, skip_context=True) - var_ok = False - self.__section_valid = False - # end try - # No else, will return None for newvar - # end if - return newvar, curr_line - - @staticmethod - def check_array_reference(local_name, var_dict, context): - """If <local_name> is an array reference, check it against - the 'dimensions' property in <var_dict>. If <local_name> is an - array reference, return it with the colons filled in with the - dictionary dimensions, otherwise, return None. - >>> MetadataSection.check_array_reference('foo', {'dimensions':['ccpp_constant_one:bar', 'ccpp_constant_one:baz']}, ParseContext(filename='foo.meta')) - - >>> MetadataSection.check_array_reference('foo', {}, ParseContext(filename='foo.meta')) - - >>> MetadataSection.check_array_reference('foo(qux', {'dimensions':['ccpp_constant_one:bar', 'ccpp_constant_one:baz']}, ParseContext(filename='foo.meta')) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: Invalid scalar reference, foo(qux, in foo.meta - >>> MetadataSection.check_array_reference('foo(qux)', {'dimensions':['ccpp_constant_one:bar', 'ccpp_constant_one:baz']}, ParseContext(filename='foo.meta')) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: foo has rank 2 but foo(qux) has 0, in foo.meta - >>> MetadataSection.check_array_reference('foo(:,qux)', {'dimensions':['ccpp_constant_one:bar', 'ccpp_constant_one:baz']}, ParseContext(filename='foo.meta')) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: foo has rank 2 but foo(:,qux) has 1, in foo.meta - >>> MetadataSection.check_array_reference('foo(:,qux)', {'foo':['ccpp_constant_one:bar', 'ccpp_constant_one:baz']}, ParseContext(filename='foo.meta')) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: Missing variable dimensions, foo(:,qux), in foo.meta - >>> MetadataSection.check_array_reference('foo(:,:,qux)', {'dimensions':['ccpp_constant_one:bar']}, ParseContext(filename='foo.meta')) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: foo has rank 1 but foo(:,:,qux) has 2, in foo.meta - >>> MetadataSection.check_array_reference('foo(:,:,qux)', {'dimensions':['ccpp_constant_one:bar','ccpp_constant_one:baz']}, ParseContext(filename='foo.meta')) - 'foo(:, :, qux)' - """ - retval = None - if check_fortran_id(local_name, var_dict, False) is None: - rmatch = FORTRAN_SCALAR_REF_RE.match(local_name) - if rmatch is None: - errmsg = 'Invalid scalar reference, {}{}' - ctx = context_string(context) - raise ParseInternalError(errmsg.format(local_name, ctx)) - # end if - rname = rmatch.group(1) - rdims = [x.strip() for x in rmatch.group(2).split(',')] - if 'dimensions' in var_dict: - vdims = [x.strip() for x in var_dict['dimensions']] - else: - errmsg = 'Missing variable dimensions, {}{}' - ctx = context_string(context) - raise ParseInternalError(errmsg.format(local_name, ctx)) - # end if - colon_rank = len([x for x in rdims if x == ':']) - if colon_rank != len(vdims): - errmsg = '{} has rank {} but {} has {}{}' - ctx = context_string(context) - raise ParseInternalError(errmsg.format(rname, len(vdims), - local_name, colon_rank, - ctx)) - # end if - sub_dims = [] - sindex = 0 - for rind in rdims: - if rind == ':': - sub_dims.append(':') - sindex += 1 - else: - sub_dims.append(rind) - # end if - # end for - retval = '{}({})'.format(rname, ', '.join(sub_dims)) - # end if - return retval - - def variable_list(self, std_vars=True, loop_vars=True, consts=True): - """Return an ordered list of the header's variables""" - return self.__variables.variable_list(recursive=False, - std_vars=std_vars, - loop_vars=loop_vars, - consts=consts) - - def find_variable(self, std_name, use_local_name=False): - """Find a variable in this header's dictionary""" - var = None - if use_local_name: - var = self.__variables.find_local_name(std_name) - else: - var = self.__variables.find_variable(std_name, any_scope=False) - # end if - return var - - def convert_dims_to_standard_names(self, var, logger=None, context=None): - """Convert the dimension elements in <var> to standard names by - by using other variables in this header. - """ - std_dims = [] - vdims = var.get_dimensions() - # Check for bad dimensions - if vdims is None: - vdim_prop = var.get_prop_value('dimensions').strip() - if vdim_prop[0] == '(': - vdim_prop = vdim_prop[1:] - # end if - if vdim_prop[-1] == ')': - vdim_prop = vdim_prop[0:-1] - # end if - vdim_strs = [x.strip() for x in vdim_prop.split(',')] - lname = var.get_prop_value('local_name') - ctx = context_string(var.context) - sep = '' - errstr = "{}{}: Invalid dimension, '{}'{}" - errmsg = '' - for vdim in vdim_strs: - if not check_fortran_id(vdim, None, False): - errmsg += errstr.format(sep, lname, vdim, ctx) - sep = '\n' - # end if - # end for - raise CCPPError("{}".format(errmsg)) - # end if - for dim in vdims: - std_dim = [] - if ':' not in dim: - # Metadata dimensions always have an explicit start - var_one = CCPP_CONSTANT_VARS.find_local_name('1') - if var_one is not None: - std = var_one.get_prop_value('standard_name') - std_dim.append(std) - # end if - # end if - for item in dim.split(':'): - try: - _ = int(item) - dvar = CCPP_CONSTANT_VARS.find_local_name(item) - if dvar is not None: - # If this integer value is a CCPP standard int, use that - dname = dvar.get_prop_value('standard_name') - else: - # Some non-standard integer value - dname = item - # end if - except ValueError as verr: - # Not an integer, try to find the standard_name - if not item: - # Naked colons are okay - dname = '' - else: - dvar = self.find_variable(item, use_local_name=True) - if dvar is not None: - dname = dvar.get_prop_value('standard_name') - else: - dname = None - # end if - # end if - if dname is None: - std = var.get_prop_value('local_name') - errmsg = f"Unknown dimension element, {item}, in {std}" - errmsg += context_string(context) - if logger is not None: - errmsg = "ERROR: " + errmsg - logger.error(errmsg.format(item, std, ctx)) - dname = unique_standard_name() - else: - raise CCPPError(errmsg) from verr - # end if - # end if - # end try - if dname is not None: - std_dim.append(dname) - else: - std_dim = None - break - # end if - # end for - if std_dim is not None: - std_dims.append(':'.join(std_dim)) - else: - break - # end if - # end for - - return std_dims - - def prop_list(self, prop_name): - """Return list of <prop_name> values for this scheme's arguments""" - return self.__variables.prop_list(prop_name) - - def section_table_mismatch(self, table_title, table_type): - """Return an error string if this arg table does not match its - metadata table parent. If they match , return an empty string.""" - mismatch = "" - # The header type must match its table's type - if self.header_type is None: - mstr = "Invalid section type, 'None'" - mismatch += mstr.format(self.header_type, table_type) - elif table_type != self.header_type: - mstr = "Section type, '{}', does not match table type, '{}'" - mismatch += mstr.format(self.header_type, table_type) - # end if - if self.header_type == SCHEME_HEADER_TYPE: - # For schemes, strip off the scheme function phase (e.g., _init) - sect_func, _, _ = CCPP_STATE_MACH.function_match(self.title) - else: - sect_func = self.title - # end if - # The Fortran parser cannot tell a scheme from a host subroutine - # Detect this and adjust - if sect_func is None: - sect_func = self.title - # end if - # The header name (minus phase) must match its table's name - if table_title != sect_func: - if mismatch: - mismatch += '\n' - # end if - mstr = "Section name, '{}', does not match table title, '{}'" - mismatch += mstr.format(self.title, table_title) - # end if - if mismatch: - mismatch += context_string(self.__pobj) - # end if - return mismatch - - @staticmethod - def variable_start(line, pobj): - """Return variable name if <line> is an interface metadata table header - """ - if line is None: - match = None - else: - match = MetadataSection.__var_start.match(line) - if match is None: - match = MetadataSection.__vref_start.match(line) - if match is not None: - name = match.group(1)+'('+match.group(2)+')' - # end if - else: - name = match.group(1) - # end if - # end if - if match is not None: - if not MetadataSection.is_scalar_reference(name): - pobj.add_syntax_err("local variable name", token=name) - name = None - # end if - else: - name = None - # end if - return name - - def write_to_file(self, filename, append=False): - """Write this metadata table to <filename>. If <append> is True, - append this table to the end of <filename>, otherwise, create - or truncate the file.""" - if append: - oflag = 'a' - else: - oflag = 'w' - # end if - with open(filename, oflag) as mfile: - mfile.write("[ccpp-arg-table]") - mfile.write(" name = {}".format(self.title)) - mfile.write(" type = {}".format(self.header_type)) - for var in self.variable_list(): - var.write_metadata(mfile) - # end for - # end with - - def to_html(self, outdir, props): - """Write html file for metadata section and return filename. - Skip metadata sections without variables""" - if not self.__variables.variable_list(): - return None - # Write table header - header = f"<tr>" - for prop in props: - header += f"<th>{prop}</th>".format(prop=prop) - header += f"</tr>\n" - # Write table contents, one row per variable - contents = "" - for var in self.__variables.variable_list(): - row = f"<tr>" - for prop in props: - value = var.get_prop_value(prop) - # Pretty-print for dimensions - if prop == 'dimensions': - value = '(' + ', '.join(value) + ')' - elif value is None: - value = f"n/a" - row += f"<td>{value}</td>".format(value=value) - row += f"</tr>\n" - contents += row - filename = os.path.join(outdir, self.title + '.html') - with open(filename,"w") as f: - f.writelines(self.__html_template__.format(title=self.title + ' argument table', - header=header, contents=contents)) - return filename - - def __repr__(self): - base = super().__repr__() - pind = base.find(' object ') - if pind >= 0: - pre = base[0:pind] - else: - pre = '<MetadataSection' - # end if - bind = base.find('at 0x') - if bind >= 0: - post = base[bind:] - else: - post = '>' - # end if - return '{} {} / {} {}'.format(pre, self.module, self.title, post) - - def __del__(self): - try: - del self.__variables - except AttributeError: - pass - - def start_context(self, with_comma=True, nodir=True): - """Return a context string for the beginning of the table""" - return context_string(self.__start_context, - with_comma=with_comma, nodir=nodir) - - @property - def title(self): - """Return the name of the metadata arg_table""" - return self.__section_title - - @property - def module(self): - """Return the module name for this header (if it exists)""" - return self.__module_name - - @property - def header_type(self): - """Return the type of structure this header documents""" - return self.__header_type - - @property - def process_type(self): - """Return the type of physical process this header documents""" - return self.__process_type - - @property - def has_variables(self): - """Convenience function for finding empty headers""" - return self.__variables - - @property - def run_env(self): - """Return this section's CCPPFrameworkEnv object""" - return self.__run_env - - @property - def valid(self): - """Return True iff we did not encounter an error creating - this section""" - return self.__section_valid - - def __str__(self): - '''Print string for MetadataSection objects''' - return "<{} {}>".format(self.__class__.__name__, self.title) - - @classmethod - def header_start(cls, line): - """Return True iff <line> is a Metadata section header (ccpp-arg-table). - """ - if (line is None) or blank_metadata_line(line): - match = None - else: - match = cls.__header_start.match(line) - # end if - return match is not None - - @staticmethod - def is_scalar_reference(test_val): - """Return True iff <test_val> refers to a Fortran scalar.""" - return check_fortran_ref(test_val, None, False) is not None - -######################################################################## diff --git a/scripts/metavar.py b/scripts/metavar.py deleted file mode 100755 index cafdbf9f..00000000 --- a/scripts/metavar.py +++ /dev/null @@ -1,2139 +0,0 @@ -#!/usr/bin/env python3 - -""" -Classes and supporting code to hold all information on CCPP metadata variables -Var: Class which holds all information on a single CCPP metadata variable -VarSpec: Class to hold a standard_name description which can include dimensions -VarAction: Base class for describing actions on variables -VarLoopSubst: Class for describing a loop substitution -VarDictionary: Class to hold all CCPP variables of a CCPP unit (e.g., suite, - scheme, host) -""" - -# Python library imports -import re -from collections import OrderedDict -# CCPP framework imports -from framework_env import CCPPFrameworkEnv -from parse_tools import check_local_name, check_fortran_type, context_string -from parse_tools import FORTRAN_SCALAR_REF_RE -from parse_tools import check_units, check_dimensions, check_cf_standard_name -from parse_tools import check_diagnostic_id, check_diagnostic_fixed -from parse_tools import check_default_value, check_valid_values -from parse_tools import check_molar_mass -from parse_tools import ParseContext, ParseSource, type_name -from parse_tools import ParseInternalError, ParseSyntaxError, CCPPError -from parse_tools import FORTRAN_CONDITIONAL_REGEX_WORDS, FORTRAN_CONDITIONAL_REGEX -from var_props import CCPP_LOOP_DIM_SUBSTS, VariableProperty, VarCompatObj -from var_props import find_horizontal_dimension, find_vertical_dimension -from var_props import standard_name_to_long_name, local_name_to_diag_name, default_kind_val - -############################################################################## - -# Dictionary of standard CCPP variables -CCPP_STANDARD_VARS = { - # Variable representing the constant integer, 1 - 'ccpp_constant_one' : - {'local_name' : '1', 'protected' : 'True', - 'standard_name' : 'ccpp_constant_one', - 'long_name' : "CCPP constant one", - 'units' : '1', 'dimensions' : '()', 'type' : 'integer'}, - 'ccpp_error_code' : - {'local_name' : 'errflg', 'standard_name' : 'ccpp_error_code', - 'long_name' : "CCPP error flag", - 'units' : '1', 'dimensions' : '()', 'type' : 'integer'}, - 'ccpp_error_message' : - {'local_name' : 'errmsg', 'standard_name' : 'ccpp_error_message', - 'long_name' : "CCPP error message", - 'units' : 'none', 'dimensions' : '()', 'type' : 'character', - 'kind' : 'len=512'}, - 'horizontal_dimension' : - {'local_name' : 'total_columns', - 'standard_name' : 'horizontal_dimension', 'units' : 'count', - 'long_name' : "total number of columns", - 'dimensions' : '()', 'type' : 'integer'}, - 'horizontal_loop_extent' : - {'local_name' : 'horz_loop_ext', - 'standard_name' : 'horizontal_loop_extent', 'units' : 'count', - 'dimensions' : '()', 'type' : 'integer'}, - 'horizontal_loop_begin' : - {'local_name' : 'horz_col_beg', - 'standard_name' : 'horizontal_loop_begin', 'units' : 'count', - 'dimensions' : '()', 'type' : 'integer'}, - 'horizontal_loop_end' : - {'local_name' : 'horz_col_end', - 'standard_name' : 'horizontal_loop_end', 'units' : 'count', - 'dimensions' : '()', 'type' : 'integer'}, - 'vertical_layer_dimension' : - {'local_name' : 'num_model_layers', - 'standard_name' : 'vertical_layer_dimension', 'units' : 'count', - 'dimensions' : '()', 'type' : 'integer'}, - 'vertical_interface_dimension' : - {'local_name' : 'num_model_interfaces', - 'standard_name' : 'vertical_interface_dimension', 'units' : 'count', - 'dimensions' : '()', 'type' : 'integer'}, - 'vertical_interface_index' : - {'local_name' : 'layer_index', - 'standard_name' : 'vertical_interface_index', 'units' : 'count', - 'dimensions' : '()', 'type' : 'integer'} -} - -# Pythonic version of a forward reference (CCPP_CONSTANT_VARS defined below) -CCPP_CONSTANT_VARS = {} -# Pythonic version of a forward reference (CCPP_VAR_LOOP_SUBST defined below) -CCPP_VAR_LOOP_SUBSTS = {} -# Loop variables only allowed during run phases -CCPP_LOOP_VAR_STDNAMES = ['horizontal_loop_extent', - 'horizontal_loop_begin', 'horizontal_loop_end', - 'vertical_layer_index', 'vertical_interface_index'] - -############################################################################### -# Used for creating template variables -_MVAR_DUMMY_RUN_ENV = CCPPFrameworkEnv(None, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - -############################################################################## - -class Var: - """ A class to hold a metadata or code variable. - Var objects should be treated as immutable. - >>> Var.get_prop('standard_name') #doctest: +ELLIPSIS - <var_props.VariableProperty object at 0x...> - >>> Var.get_prop('standard') - - >>> Var.get_prop('type').is_match('type') - True - >>> Var.get_prop('type').is_match('long_name') - False - >>> Var.get_prop('type').valid_value('character') - 'character' - >>> Var.get_prop('type').valid_value('char') - - >>> Var.get_prop('long_name').valid_value('hi mom') - 'hi mom' - >>> Var.get_prop('dimensions').valid_value('hi mom') - - >>> Var.get_prop('dimensions').valid_value(['Bob', 'Ray']) - ['Bob', 'Ray'] - >>> Var.get_prop('active') #doctest: +ELLIPSIS - <var_props.VariableProperty object at 0x...> - >>> Var.get_prop('active').get_default_val({}) - '.true.' - >>> Var.get_prop('active').valid_value('flag_for_aerosol_physics') - 'flag_for_aerosol_physics' - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'DDT', ParseContext()), _MVAR_DUMMY_RUN_ENV).get_prop_value('active') - '.true.' - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in', 'active' : 'child_is_home==.true.'}, ParseSource('vname', 'DDT', ParseContext()), _MVAR_DUMMY_RUN_ENV).get_prop_value('active') - 'child_is_home==.true.' - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV).get_prop_value('long_name') - 'Hi mom' - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV).get_prop_value('intent') - 'in' - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV).get_prop_value('units') - 'm s-1' - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV).get_prop_value('units') #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Required property, 'units', missing, in <standard input> - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : ' ', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV).get_prop_value('units') #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: foo: ' ' is not a valid unit, in <standard input> - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'ttype' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Invalid metadata variable property, 'ttype', in <standard input> - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Required property, 'units', missing, in <standard input> - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'inout', 'protected' : '.true.'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: foo is marked protected but is intent inout, at <standard input>:1 - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'ino'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Invalid intent variable property, 'ino', at <standard input>:1 - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in', 'optional' : 'false'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV) #doctest: +ELLIPSIS - <metavar.Var hi_mom: foo at 0x...> - - # Check that two variables that differ in their units - m vs km - are compatible - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm', \ - 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, \ - ParseSource('vname', 'SCHEME', ParseContext()), \ - _MVAR_DUMMY_RUN_ENV).compatible(Var({'local_name' : 'bar', \ - 'standard_name' : 'hi_mom', 'units' : 'km', \ - 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, \ - ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV), \ - _MVAR_DUMMY_RUN_ENV) #doctest: +ELLIPSIS - <var_props.VarCompatObj object at ...> - """ - - ## Prop lists below define all the allowed CCPP Metadata attributes - - # __spec_props are for variables defined in a specification - __spec_props = [VariableProperty('local_name', str, - check_fn_in=check_local_name), - VariableProperty('standard_name', str, - check_fn_in=check_cf_standard_name), - VariableProperty('long_name', str, optional_in=True, - default_fn_in=standard_name_to_long_name), - VariableProperty('diagnostic_name', str, optional_in=True, - default_fn_in=local_name_to_diag_name, - check_fn_in=check_diagnostic_id), - VariableProperty('units', str, - check_fn_in=check_units), - VariableProperty('dimensions', list, - check_fn_in=check_dimensions), - VariableProperty('type', str, - check_fn_in=check_fortran_type), - VariableProperty('kind', str, - optional_in=True, - default_fn_in=default_kind_val), - VariableProperty('state_variable', bool, - optional_in=True, default_in=False), - VariableProperty('protected', bool, - optional_in=True, default_in=False), - VariableProperty('allocatable', bool, - optional_in=True, default_in=False), - VariableProperty('diagnostic_name_fixed', str, - optional_in=True, default_in='', - check_fn_in=check_diagnostic_fixed), - VariableProperty('default_value', str, - optional_in=True, default_in='', - check_fn_in=check_default_value), - VariableProperty('persistence', str, optional_in=True, - valid_values_in=['timestep', 'run'], - default_in='timestep'), - VariableProperty('active', str, optional_in=True, - default_in='.true.'), - VariableProperty('polymorphic', bool, optional_in=True, - default_in=False), - VariableProperty('top_at_one', bool, optional_in=True, - default_in=False), - VariableProperty('optional', bool, optional_in=True, - default_in=False), - VariableProperty('target', bool, optional_in=True, - default_in=False)] - -# XXgoldyXX: v debug only - __to_add = VariableProperty('valid_values', str, - optional_in=True, default_in='', - check_fn_in=check_valid_values) -# XXgoldyXX: ^ debug only - - # __var_props contains properties which are not in __spec_props - __var_props = [VariableProperty('intent', str, - valid_values_in=['in', 'out', 'inout'])] - - # __constituent_props contains properties associated only with constituents - # Note that all constituent properties must be optional and contain either - # a default value or default function. - __constituent_props = [VariableProperty('advected', bool, - optional_in=True, default_in=False), - VariableProperty('molar_mass', float, - optional_in=True, default_in=0.0, - check_fn_in=check_molar_mass), - VariableProperty('constituent', bool, - optional_in=True, default_in=False)] - - __constituent_prop_dict = {x.name : x for x in __constituent_props} - - # __no_metadata_props__ contains properties to omit from metadata - __no_metadata_props__ = ['local_name'] - - __spec_propdict = {p.name : p for p in __spec_props} - __var_propdict = {p.name : p for p in __spec_props + __var_props} - __required_spec_props = list() - __required_var_props = list() - for p in __spec_props: - __var_propdict[p.name] = p - if not p.optional: - __required_spec_props.append(p.name) - __required_var_props.append(p.name) - # end if - # end for - for p in __var_props: -# XXgoldyXX: v why? -# __spec_propdict[p.name] = p -# XXgoldyXX: ^ why? -# __var_propdict[p.name] = p - if not p.optional: - __required_var_props.append(p.name) - # end if - # end for - __var_propdict.update({p.name : p for p in __constituent_props}) - # All constituent props are optional so no check - - def __init__(self, prop_dict, source, run_env, context=None, - clone_source=None, fortran_imports=None): - """Initialize a new Var object. - If <prop_dict> is really a Var object, use that object's prop_dict. - If this Var object is a clone, record the original Var object - for reference - <source> is a ParseSource object describing the source of this Var. - <run_env> is the CCPPFrameworkEnv object for this framework run. - <context> is a ParseContext object - <clone_source> is a Var object. If provided, it is used as the original - source of a cloned variable. - """ - self.__parent_var = None # for array references - self.__children = list() # This Var's array references - self.__clone_source = clone_source - self.__run_env = run_env - if isinstance(prop_dict, Var): - prop_dict = prop_dict.copy_prop_dict() - # end if - if source.ptype == 'scheme': - self.__required_props = Var.__required_var_props -# XXgoldyXX: v don't fill in default properties? -# mstr_propdict = Var.__var_propdict -# XXgoldyXX: ^ don't fill in default properties? - else: - self.__required_props = Var.__required_spec_props -# XXgoldyXX: v don't fill in default properties? - mstr_propdict = Var.__spec_propdict -# XXgoldyXX: ^ don't fill in default properties? - # end if - self.__source = source - # Grab a frozen copy of the context - if context is None: - self._context = ParseContext(context=source.context) - else: - self._context = context - # end if - # First, check the input - if 'ddt_type' in prop_dict: - # Special case to bypass normal type rules - if 'type' not in prop_dict: - prop_dict['type'] = prop_dict['ddt_type'] - # end if - if 'units' not in prop_dict: - prop_dict['units'] = "" - # end if - # DH* To investigate later: Why is the DDT type - # copied into the kind attribute? Can we remove this? - prop_dict['kind'] = prop_dict['ddt_type'] - del prop_dict['ddt_type'] - self.__intrinsic = False - else: - self.__intrinsic = True - # end if - for key in prop_dict: - if Var.get_prop(key) is None: - raise ParseSyntaxError("Invalid metadata variable property, '{}'".format(key), context=self.context) - # end if - # end for - # Make sure required properties are present - for propname in self.__required_props: - if propname not in prop_dict: - emsg = "Required property, '{}', missing" - raise ParseSyntaxError(emsg.format(propname), - context=self.context) - # end if - # end for - # Check for any mismatch - if ('protected' in prop_dict) and ('intent' in prop_dict): - if (prop_dict['intent'].lower() != 'in') and prop_dict['protected']: - emsg = "{} is marked protected but is intent {}" - raise ParseSyntaxError(emsg.format(prop_dict['local_name'], - prop_dict['intent']), - context=self.context) - # end if - # end if - # Look for any constituent properties - self.__is_constituent = False - for name, prop in Var.__constituent_prop_dict.items(): - if (name in prop_dict) and \ - (prop_dict[name] != prop.get_default_val(prop_dict, - context=self.context)): - self.__is_constituent = True - break - # end if - # end for - # Steal dict from caller - self._prop_dict = prop_dict - # Make sure all the variable values are valid - try: - for prop_name, prop_val in self.var_properties(): - prop = Var.get_prop(prop_name) - _ = prop.valid_value(prop_val, - prop_dict=self._prop_dict, error=True) - # end for - except CCPPError as cperr: - # Raise this error unless it represents an imported DDT type - if ((not fortran_imports) or (prop_name != 'type') or - (prop_val not in fortran_imports)): - lname = self._prop_dict['local_name'] - emsg = "{}: {}" - raise ParseSyntaxError(emsg.format(lname, cperr), - context=self.context) from cperr - # end if - # end try - - def compatible(self, other, run_env, is_tend=False): - """Return a VarCompatObj object which describes the equivalence, - compatibility, or incompatibility between <self> and <other>. - """ - # We accept character(len=*) as compatible with - # character(len=INTEGER_VALUE) - stype = self.get_prop_value('type') - skind = self.get_prop_value('kind') - sunits = self.get_prop_value('units') - sstd_name = self.get_prop_value('standard_name') - sloc_name = self.get_prop_value('local_name') - stopp = self.get_prop_value('top_at_one') - sdims = self.get_dimensions() - otype = other.get_prop_value('type') - okind = other.get_prop_value('kind') - ounits = other.get_prop_value('units') - ostd_name = other.get_prop_value('standard_name') - oloc_name = other.get_prop_value('local_name') - otopp = other.get_prop_value('top_at_one') - odims = other.get_dimensions() - compat = VarCompatObj(sstd_name, stype, skind, sunits, sdims, sloc_name, stopp, - ostd_name, otype, okind, ounits, odims, oloc_name, otopp, - run_env, - v1_context=self.context, v2_context=other.context, is_tend=is_tend) - if (not compat) and (run_env.logger is not None): - incompat_str = compat.incompat_reason - if incompat_str is not None: - run_env.logger.info('{}'.format(incompat_str)) - # end if (no else) - # end if - return compat - - def adjust_intent(self, src_var): - """Add an intent to this Var or adjust its existing intent. - Note: An existing intent can only be adjusted to 'inout' - """ - if 'intent' in self._prop_dict: - my_intent = self.get_prop_value('intent') - else: - my_intent = None - # end if - sv_intent = src_var.get_prop_value('intent') - if not sv_intent: - sv_intent = 'in' - # end if - if sv_intent in ['inout', 'out'] and self.get_prop_value('protected'): - lname = self.get_prop_value('local_name') - lctx = context_string(self.context) - emsg = "Attempt to set intent of {}{} to {}, only 'in' allowed " - emsg += "for 'protected' variable." - if src_var: - slname = src_var.get_prop_value('local_name') - sctx = context_string(src_var.context) - emsg += "\nintent source: {}{}".format(slname, sctx) - # end if - raise CCPPError(emsg.format(lname, lctx, sv_intent)) - # end if (else, no error) - if my_intent: - if my_intent != sv_intent: - self._prop_dict['intent'] = 'inout' - # end if (no else, intent is okay) - else: - self._prop_dict['intent'] = sv_intent - # end if - - @staticmethod - def get_prop(name, spec_type=None): - """Return VariableProperty object for <name> or None""" - prop = None - if (spec_type is None) and (name in Var.__var_propdict): - prop = Var.__var_propdict[name] - elif (spec_type is not None) and (name in Var.__spec_propdict): - prop = Var.__spec_propdict[name] - # end if (else prop = None) - return prop - - def var_properties(self): - """Return an iterator for this Var's property dictionary""" - return self._prop_dict.items() - - def copy_prop_dict(self, subst_dict=None): - """Create a copy of our prop_dict, possibly substituting properties - from <subst_dict>.""" - cprop_dict = {} - # Start with a straight copy of this variable's prop_dict - for prop, val in self.var_properties(): - cprop_dict[prop] = val - # end for - # Now add or substitute properties from <subst_dict> - if subst_dict: - for prop in subst_dict.keys(): - cprop_dict[prop] = subst_dict[prop] - # end for - # end if - # Special key for creating a copy of a DDT (see Var.__init__) - if self.is_ddt(): - cprop_dict['ddt_type'] = cprop_dict['type'] - # end if - return cprop_dict - - def clone(self, subst_dict=None, remove_intent=False, - source_name=None, source_type=None, context=None): - """Create a clone of this Var object with properties from <subst_dict> - overriding this variable's properties. <subst_dict> may also be - a string in which case only the local_name property is changed - (to the value of the <subst_dict> string). - If <remove_intent> is True, remove the 'intent' property, if present. - This can be used to promote a variable to module level. - The optional <source_name>, <source_type>, and <context> inputs - allow the clone to appear to be coming from a designated source, - by default, the source and type are the same as this Var (self). - """ - if isinstance(subst_dict, str): - subst_dict = {'local_name':subst_dict} - elif subst_dict is None: - subst_dict = {} - # end if - cprop_dict = self.copy_prop_dict(subst_dict=subst_dict) - if remove_intent and ('intent' in cprop_dict): - del cprop_dict['intent'] - # end if - if source_name is None: - source_name = self.source.name - # end if - if source_type is None: - source_type = self.source.ptype - # end if - if context is None: - context = self._context - # end if - psource = ParseSource(source_name, source_type, context) - - return Var(cprop_dict, psource, self.run_env, clone_source=self) - - def get_prop_value(self, name): - """Return the value of key, <name> if <name> is in this variable's - property dictionary. - If <name> is not in the prop dict but does have a <default_fn_in> - property, return the value specified by calling that function. - Otherwise, return None - """ - if name in self._prop_dict: - pvalue = self._prop_dict[name] - elif name in Var.__var_propdict: - vprop = Var.__var_propdict[name] - if vprop.optional: - pvalue = vprop.get_default_val(self._prop_dict, - context=self.context) - else: - pvalue = None - # end if - else: - pvalue = None - # end if - return pvalue - - def handle_array_ref(self): - """If this Var's local_name is an array ref, add in the array - reference indices to the Var's dimensions. - Return the (stripped) local_name and the full dimensions. - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real',}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).handle_array_ref() - ('foo', []) - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '(ccpp_constant_one:dim1)', 'type' : 'real',}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).handle_array_ref() - ('foo', ['ccpp_constant_one:dim1']) - >>> Var({'local_name' : 'foo(:,:,bar)', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '(ccpp_constant_one:dim1,ccpp_constant_one:dim2)', 'type' : 'real',}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).handle_array_ref() - ('foo', ['ccpp_constant_one:dim1', 'ccpp_constant_one:dim2', 'bar']) - >>> Var({'local_name' : 'foo(bar,:)', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '(ccpp_constant_one:dim1)', 'type' : 'real',}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).handle_array_ref() - ('foo', ['bar', 'ccpp_constant_one:dim1']) - >>> Var({'local_name' : 'foo(bar)', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '(ccpp_constant_one:dim1)', 'type' : 'real',}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).handle_array_ref() #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: Call dims mismatch for foo(bar), not enough colons - >>> Var({'local_name' : 'foo(:,bar,:)', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '(ccpp_constant_one:dim1)', 'type' : 'real',}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).handle_array_ref() #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: Call dims mismatch for foo(:,bar,:), not enough dims - >>> Var({'local_name' : 'foo(:,:,bar)', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '(ccpp_constant_one:dim1)', 'type' : 'real',}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).handle_array_ref() #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: Call dims mismatch for foo(:,:,bar), not enough dims - >>> Var({'local_name' : 'foo(:,bar)', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '(ccpp_constant_one:dim1,ccpp_constant_one:dim2)', 'type' : 'real',}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).handle_array_ref() #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: Call dims mismatch for foo(:,bar), too many dims - """ - dimlist = self.get_dimensions() - aref = self.array_ref() - if aref is not None: - lname = aref.group(1) - # Substitute dimensions for colons in array reference - sdimlist = dimlist - num_dims = len(sdimlist) - dimlist = [x.strip() for x in aref.group(2).split(',')] - num_colons = sum(dim == ':' for dim in dimlist) - cind = 0 - if num_dims > num_colons: - emsg = 'Call dims mismatch for {}, not enough colons' - lname = self.get_prop_value('local_name') - raise CCPPError(emsg.format(lname)) - # end if - for dind, dim in enumerate(dimlist): - if dim == ':': - if cind >= num_dims: - emsg = 'Call dims mismatch for {}, not enough dims' - lname = self.get_prop_value('local_name') - raise CCPPError(emsg.format(lname)) - # end if - dimlist[dind] = sdimlist[cind] - cind += 1 - # end if - # end for - if cind < num_colons: - emsg = 'Call dims mismatch for {}, too many dims' - lname = self.get_prop_value('local_name') - raise CCPPError(emsg.format(lname)) - # end if - else: - lname = self.get_prop_value('local_name') - # end if - return lname, dimlist - - def call_dimstring(self, var_dicts=None, - explicit_dims=False, loop_subst=False): - """Return the dimensions string for a variable call. - If <var_dict> is present, find and substitute a local_name for - each standard_name in this variable's dimensions. - If <var_dict> is not present, return a colon for each dimension. - If <explicit_dims> is True, include the variable's dimensions. - If <loop_subst> is True, apply a loop substitution, if found for any - missing dimension. - """ - emsg = '' - _, dims = self.handle_array_ref() - if var_dicts is not None: - dimlist = [] - sepstr = '' - for dim in dims: - # Decide whether to list all dimensions or to replace - # a range with a colon. - dstdnames = dim.split(':') - add_dims = explicit_dims or (len(dstdnames) == 1) - dvar = None - if add_dims and loop_subst: - for vdict in var_dicts: - dvar = vdict.find_loop_dim_match(dim) - if dvar is not None: - break - # end if - # end for - if dvar: - dimlist.append(dvar) - # end if - if (not dvar) and add_dims: - dnames = [] - for stdname in dstdnames: - for vdict in var_dicts: - dvar = vdict.find_variable(standard_name=stdname, - any_scope=False) - if dvar is not None: - break - # end if - # end for - if dvar: - # vdict is the dictionary where <dvar> was found - dnames.append(dvar.call_string(vdict)) - # end if - if not dvar: - emsg += sepstr + "No variable found in " - vnames = [x.name for x in var_dicts] - if len(vnames) > 2: - vstr = ', '.join(vnames[:-1]) - vstr += ', or {}'.format(vnames[-1]) - elif len(vnames) > 1: - vstr = ' or '.join(vnames) - else: - vstr = vnames[0] - # end if - emsg += "{} for dimension '".format(vstr) - emsg += stdname + "' in {vlnam}" - sepstr = '\n' - # end if - # end for - dimlist.append(':'.join(dnames)) - elif not add_dims: - dimlist.append(':') - # end if (no else needed, we must have found loop substitution) - # end for - else: - dimlist = [':']*len(dims) - # end if - if dimlist: - dimstr = '(' + ','.join(dimlist) + ')' - else: - dimstr = '' # It ends up being a scalar reference - # end if - if emsg: - ctx = context_string(self.context) - emsg += "{ctx}" - lname = self.get_prop_value('local_name') - raise CCPPError(emsg.format(vlnam=lname, ctx=ctx)) - # end if - return dimstr - - def call_string(self, var_dict, loop_vars=None): - """Construct the actual argument string for this Var by translating - standard names to local names. - String includes array bounds unless loop_vars is None. - if <loop_vars> is not None, look there first for array bounds, - even if usage requires a loop substitution. - """ - if loop_vars is None: - call_str = self.get_prop_value('local_name') - # Look for dims in case this is an array selection variable - dind = call_str.find('(') - if dind > 0: - dimstr = call_str[dind+1:].rstrip()[:-1] - dims = [x.strip() for x in dimstr.split(',')] - call_str = call_str[:dind].strip() - else: - dims = None - # end if - else: - call_str, dims = self.handle_array_ref() - # end if - if dims: - call_str += '(' - dsep = '' - for dim in dims: - if loop_vars: - lname = loop_vars.find_loop_dim_match(dim) - else: - lname = None - # end if - if lname is None: - isep = '' - lname = "" - for item in dim.split(':'): - if item: - dvar = var_dict.find_variable(standard_name=item, - any_scope=False) - if dvar is None: - try: - dval = int(item) - iname = item - except ValueError: - iname = None - # end try - else: - iname = dvar.call_string(var_dict, - loop_vars=loop_vars) - # end if - else: - iname = '' - # end if - if iname is not None: - lname = lname + isep + iname - isep = ':' - else: - errmsg = 'No local variable {} in {}{}' - ctx = context_string(self.context) - dname = var_dict.name - raise CCPPError(errmsg.format(item, dname, ctx)) - # end if - # end for - # end if - if lname is not None: - call_str += dsep + lname - dsep = ', ' - else: - errmsg = 'Unable to convert {} to local variables in {}{}' - ctx = context_string(self.context) - raise CCPPError(errmsg.format(dim, var_dict.name, ctx)) - # end if - # end for - call_str += ')' - # end if - return call_str - - def valid_value(self, prop_name, test_value=None, error=False): - """Return a valid version of <test_value> if it is a valid value - for the property, <prop_name>. - If <test_value> is not valid, return None or raise an exception, - depending on the value of <error>. - If <test_value> is None, use the current value of <prop_name>. - """ - vprop = Var.get_prop(prop_name) - if vprop is not None: - if test_value is None: - test_val = self.get_prop_value(prop_name) - # end if - valid = vprop.valid_value(test_val, - prop_dict=self._prop_dict, error=error) - else: - valid = None - errmsg = 'Invalid variable property, {}' - raise ParseInternalError(errmsg.format(prop_name)) - # end if - return valid - - def array_ref(self, local_name=None): - """If this Var's local_name is an array reference, return a - Fortran array reference regexp match. - Otherwise, return None""" - if local_name is None: - local_name = self.get_prop_value('local_name') - # end if - match = FORTRAN_SCALAR_REF_RE.match(local_name) - return match - - def intrinsic_elements(self, check_dict=None, ddt_lib=None): - """Return a list of the standard names of this Var object's 'leaf' - intrinsic elements or this Var object's standard name if it is an - intrinsic 'leaf' variable. - If this Var object cannot be reduced to one or more intrinsic 'leaf' - variables (e.g., a DDT Var with no named elements), return None. - A 'leaf' intrinsic Var is a Var of intrinsic Fortran type which has - no children. If a Var has children, those children will be searched - to find leaves. If a Var is a DDT, its named elements are searched. - If <check_dict> is not None, it is checked for children if none are - found in this variable (via finding a variable in <check_dict> with - the same standard name). - Currently, an array of DDTs is not processed (return None) since - Fortran does not support a way to reference those elements. - """ - element_names = None - if self.is_ddt(): - dtitle = self.get_prop_value('type') - if ddt_lib and (dtitle in ddt_lib): - element_names = [] - ddt_def = ddt_lib[dtitle] - for dvar in ddt_def.variable_list(): - delems = dvar.intrinsic_elements(check_dict=check_dict, - ddt_lib=ddt_lib) - if delems: - element_names.extend(delems) - # end if - # end for - if not element_names: - element_names = None - # end if - else: - errmsg = f'No ddt_lib or ddt {dtitle} not in ddt_lib' - raise CCPPError(errmsg) - # end if - # end if - children = self.children() - if (not children) and check_dict: - stdname = self.get_prop_value("standard_name") - pvar = check_dict.find_variable(standard_name=stdname, - any_scope=True) - if pvar: - children = pvar.children() - # end if - # end if - if children: - element_names = list() - for child in children: - child_elements = child.intrinsic_elements() - if isinstance(child_elements, str): - child_elements = [child_elements] - # end if - if child_elements: - for elem in child_elements: - if elem: - element_names.append(elem) - # end if - # end for - # end if - # end for - else: - element_names = self.get_prop_value('standard_name') - # end if - return element_names - - @classmethod - def constituent_property_names(cls): - """Return a list of the names of constituent properties""" - return Var.__constituent_prop_dict.keys() - - @property - def parent(self): - """Return this variable's parent variable (or None)""" - return self.__parent_var - - @parent.setter - def parent(self, parent_var): - """Set this variable's parent if not already set""" - if self.__parent_var is not None: - emsg = 'Attempting to set parent for {} but parent already set' - lname = self.get_prop_value('local_name') - raise ParseInternalError(emsg.format(lname)) - # end if - if isinstance(parent_var, Var): - self.__parent_var = parent_var - parent_var.add_child(self) - else: - emsg = 'Attempting to set parent for {}, bad parent type, {}' - lname = self.get_prop_value('local_name') - raise ParseInternalError(emsg.format(lname, type_name(parent_var))) - # end if - - def add_child(self, cvar): - """Add <cvar> as a child of this Var object""" - if cvar not in self.__children: - self.__children.append(cvar) - # end if - - def children(self): - """Return an iterator over this object's children or None if the - object has no children.""" - children = self.__children - if not children: - pvar = self - while (not children) and pvar.clone_source: - pvar = pvar.clone_source - children = pvar.children() - # end while - # end if - return iter(children) if children else None - - @property - def var(self): - "Return this object (base behavior for derived classes such as VarDDT)" - return self - - @property - def context(self): - """Return this variable's parsed context""" - return self._context - - @property - def source(self): - """Return the source object for this variable""" - return self.__source - - @source.setter - def source(self, new_source): - """Reset this Var's source if <new_source> seems legit""" - if isinstance(new_source, ParseSource): - self.__source = new_source - else: - errmsg = 'Attemping to set source of {} ({}) to "{}"' - stdname = self.get_prop_value('standard_name') - lname = self.get_prop_value('local_name') - raise ParseInternalError(errmsg.format(stdname, lname, new_source)) - # end if - - @property - def clone_source(self): - """Return this Var object's clone source (or None)""" - return self.__clone_source - - @property - def host_interface_var(self): - """True iff self is included in the host model interface calls""" - return self.source.ptype == 'host' - - @property - def run_env(self): - """Return the CCPPFrameworkEnv object used to create this Var object.""" - return self.__run_env - - def get_dimensions(self): - """Return a list with the variable's dimension strings""" - dims = self.valid_value('dimensions') - return dims - - def get_dim_stdnames(self, include_constants=True): - """Return a set of all the dimension standard names for this Var""" - dimset = set() - for dim in self.get_dimensions(): - for name in dim.split(':'): - # Weed out the integers - try: - _ = int(name) - except ValueError: - # Not an integer, maybe add it - if include_constants or (not name in CCPP_CONSTANT_VARS): - dimset.add(name) - # end if - # end try - # end for - # end for - return dimset - - def get_rank(self): - """Return the variable's rank (zero for scalar)""" - dims = self.get_dimensions() - return len(dims) - - def has_horizontal_dimension(self, dims=None): - """Return horizontal dimension standard name string for - <self> or <dims> (if present) if a horizontal dimension is - present in the list""" - if dims is None: - vdims = self.get_dimensions() - else: - vdims = dims - # end if - return find_horizontal_dimension(vdims)[0] - - def has_vertical_dimension(self, dims=None): - """Return vertical dimension standard name string for - <self> or <dims> (if present) if a vertical dimension is - present in the list""" - if dims is None: - vdims = self.get_dimensions() - else: - vdims = dims - # end if - return find_vertical_dimension(vdims)[0] - - def conditional(self, vdicts): - """Convert conditional expression from active attribute - (i.e. in standard name format) to local names based on vdict. - Return conditional and a list of variables needed to evaluate - the conditional. - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real',}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).conditional([{}]) - ('.true.', []) - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'active' : 'False'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).conditional([VarDictionary('bar', _MVAR_DUMMY_RUN_ENV, variables={})]) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - Exception: Cannot find variable 'false' for generating conditional for 'False' - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'active' : 'mom_gone'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).conditional([ VarDictionary('bar', _MVAR_DUMMY_RUN_ENV, variables=[Var({'local_name' : 'bar', 'standard_name' : 'mom_home', 'units' : '', 'dimensions' : '()', 'type' : 'logical'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV)]) ]) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - Exception: Cannot find variable 'mom_gone' for generating conditional for 'mom_gone' - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'active' : 'mom_home'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).conditional([ VarDictionary('bar', _MVAR_DUMMY_RUN_ENV, variables=[Var({'local_name' : 'bar', 'standard_name' : 'mom_home', 'units' : '', 'dimensions' : '()', 'type' : 'logical'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV)]) ])[0] - 'bar' - >>> len(Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'active' : 'mom_home'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).conditional([ VarDictionary('bar', _MVAR_DUMMY_RUN_ENV, variables=[Var({'local_name' : 'bar', 'standard_name' : 'mom_home', 'units' : '', 'dimensions' : '()', 'type' : 'logical'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV)]) ])[1]) - 1 - """ - - active = self.get_prop_value('active') - conditional = '' - vars_needed = [] - - # Find all words in the conditional, for each of them look - # for a matching standard name in the list of known variables - items = FORTRAN_CONDITIONAL_REGEX.findall(active) - for item in items: - item = item.lower() - if item in FORTRAN_CONDITIONAL_REGEX_WORDS: - conditional += item - else: - # Keep integers - try: - int(item) - conditional += item - except ValueError: - dvar = None - for vdict in vdicts: - dvar = vdict.find_variable(standard_name=item, any_scope=True) # or any_scope=False ? - if dvar: - break - if not dvar: - raise Exception(f"Cannot find variable '{item}' for generating conditional for '{active}'") - conditional += dvar.get_prop_value('local_name') - vars_needed.append(dvar) - return (conditional, vars_needed) - - def write_def(self, outfile, indent, wdict, allocatable=False, target=False, - dummy=False, add_intent=None, extra_space=0, public=False): - """Write the definition line for the variable to <outfile>. - If <dummy> is True, include the variable's intent. - If <dummy> is True but the variable has no intent, add the - intent indicated by <add_intent>. This is intended for host model - variables and it is an error to not pass <add_intent> if <dummy> - is True and the variable has no intent property.""" - stdname = self.get_prop_value('standard_name') - if stdname in CCPP_CONSTANT_VARS: - # There is no declaration line for a constant - return - # end if - if self.is_ddt(): - vtype = 'type' - else: - vtype = self.get_prop_value('type') - # end if - kind = self.get_prop_value('kind') - name = self.get_prop_value('local_name') - aref = self.array_ref(local_name=name) - if aref is not None: - name = aref.group(1) - # end if - dims = self.get_dimensions() - if dims: - if allocatable or dummy: - dimstr = '(:' + ',:'*(len(dims) - 1) + ')' - else: - dimstr = self.call_dimstring(var_dicts=[wdict]) - else: - dimstr = '' - # end if - protected = self.get_prop_value('protected') - polymorphic = self.get_prop_value('polymorphic') - if dummy: - intent = self.get_prop_value('intent') - else: - intent = None - # end if - if protected and allocatable: - errmsg = "Cannot create allocatable variable from protected, {}" - raise CCPPError(errmsg.format(name)) - # end if - if dummy and (intent is None): - if add_intent is not None: - intent = add_intent - else: - errmsg = f"<add_intent> is missing for dummy argument, {name}" - raise CCPPError(errmsg) - # end if - # end if - optional = self.get_prop_value('optional') - if protected and dummy: - intent_str = 'intent(in) ' - elif allocatable: - if dimstr or polymorphic: - intent_str = 'allocatable ' - if target: - intent_str = 'allocatable,' - intent_str += ' target' - else: - intent_str = ' '*13 - # end if - elif intent is not None: - alloval = self.get_prop_value('allocatable') - if (intent.lower()[-3:] == 'out') and alloval: - intent_str = f"allocatable, intent({intent}){' '*(5 - len(intent))}" - elif optional: - intent_str = f"intent({intent}),{' '*(5 - len(intent))}" - intent_str += 'target, optional ' - else: - intent_str = f"intent({intent}){' '*(5 - len(intent))}" - # end if - elif not dummy: - intent_str = ' '*20 - else: - intent_str = ' '*13 - # end if - if intent_str.strip(): - comma = ',' - else: - comma = ' ' - # end if - if self.get_prop_value('target'): - targ = ", target" - else: - targ = "" - # end if - comma = targ + comma - extra_space -= len(targ) - if self.is_ddt(): - if polymorphic: - dstr = "class({kind}){cspace}{intent} :: {name}{dims}" - cspace = comma + ' '*(extra_space + 12 - len(kind)) - else: - dstr = "type({kind}){cspace}{intent} :: {name}{dims}" - cspace = comma + ' '*(extra_space + 13 - len(kind)) - # end if - else: - if kind: - dstr = "{type}({kind}){cspace}{intent} :: {name}{dims}" - cspace = comma + ' '*(extra_space + 17 - len(vtype) - len(kind)) - else: - dstr = "{type}{cspace}{intent} :: {name}{dims}" - cspace = comma + ' '*(extra_space + 19 - len(vtype)) - # end if - # end if - outfile.write(dstr.format(type=vtype, kind=kind, intent=intent_str, - name=name, dims=dimstr, cspace=cspace, - sname=stdname), indent) - - def write_ptr_def(self, outfile, indent, name, kind, dimstr, vtype, extra_space=0): - """Write the definition line for local null pointer declaration to <outfile>.""" - comma = ', ' - if kind: - dstr = "{type}({kind}){cspace}pointer :: {name}{dims}{cspace2} => null()" - cspace = comma + ' '*(extra_space + 20 - len(vtype) - len(kind)) - cspace2 = ' '*(20 -len(name) - len(dimstr)) - else: - dstr = "{type}{cspace}pointer :: {name}{dims}{cspace2} => null()" - cspace = comma + ' '*(extra_space + 22 - len(vtype)) - cspace2 = ' '*(20 -len(name) - len(dimstr)) - # end if - outfile.write(dstr.format(type=vtype, kind=kind, name=name, dims=dimstr, - cspace=cspace, cspace2=cspace2), indent) - - def is_ddt(self): - """Return True iff <self> is a DDT type.""" - return not self.__intrinsic - - def is_constituent(self): - """Return True iff <self> is a constituent variable.""" - return self.__is_constituent - - def __str__(self): - """Print representation or string for Var objects""" - return "<Var {standard_name}: {local_name}>".format(**self._prop_dict) - - def __repr__(self): - """Object representation for Var objects""" - base = super().__repr__() - pind = base.find(' object ') - if pind >= 0: - pre = base[0:pind] - else: - pre = '<Var' - # end if - bind = base.find('at 0x') - if bind >= 0: - post = base[bind:] - else: - post = '>' - # end if - return '{} {}: {} {}'.format(pre, self._prop_dict['standard_name'], - self._prop_dict['local_name'], post) - -############################################################################### - -class FortranVar(Var): - """A class to hold the metadata for a Fortran variable which can - contain properties not used in CCPP metadata. - """ - - __fortran_props = [VariableProperty('optional', bool, - optional_in=True, default_in=False)] - - def __init__(self, prop_dict, source, run_env, context=None, - clone_source=None, fortran_imports=None): - """Initialize a FortranVar object. - """ - - # Remove and save any Fortran-only properties - save_dict = {} - for prop in self.__fortran_props: - if prop.name in prop_dict: - save_dict[prop.name] = prop_dict[prop.name] - del prop_dict[prop.name] - # end if - # end for - # Initialize Var - super().__init__(prop_dict, source, run_env, context=context, - clone_source=clone_source, - fortran_imports=fortran_imports) - # Now, restore the saved properties - for prop in save_dict: - self._prop_dict[prop] = save_dict[prop] - # end for - - -############################################################################### - -class VarSpec: - """A class to hold a standard_name description of a variable. - A scalar variable is just a standard name while an array also - contains a comma-separated list of dimension standard names in parentheses. - """ - - def __init__(self, var): - """Initialize the common properties of this VarSpec-based object""" - self.__name = var.get_prop_value('standard_name') - self.__dims = var.get_dimensions() - if not self.__dims: - self.__dims = None - # end if - - @property - def name(self): - """Return the name of this VarSpec-based object""" - return self.__name - - def get_dimensions(self): - """Return the dimensions of this VarSpec-based object.""" - rdims = self.__dims - return rdims - - def __repr__(self): - """Return a representation of this object""" - if self.__dims is not None: - repr_str = f"{self.__name}({', '.join(self.__dims)})" - else: - repr_str = self.__name - # end if - return repr_str - -############################################################################### - -__CCPP_PARSE_CONTEXT = ParseContext(filename='metavar.py') - -############################################################################### - -def ccpp_standard_var(std_name, source_type, run_env, - context=None, intent='out'): - """If <std_name> is a CCPP standard variable name, return a variable - with that name. - Otherwise return None. - """ - if std_name in CCPP_STANDARD_VARS: - # Copy the dictionary because Var can change it - vdict = dict(CCPP_STANDARD_VARS[std_name]) - if context is None: - psource = ParseSource('ccpp_standard_vars', source_type, - __CCPP_PARSE_CONTEXT) - else: - psource = ParseSource('ccpp_standard_vars', source_type, context) - # end if - if source_type.lower() == 'scheme': - vdict['intent'] = intent - # end if - newvar = Var(vdict, psource, run_env) - else: - newvar = None - # end if - return newvar - -############################################################################### - -class VarAction: - """A base class for variable actions such as loop substitutions or - temporary variable handling.""" - - def __init__(self): - """Initialize this action (nothing to do)""" - # pass # Nothing general here yet - - def add_local(self, vadict, source): - """Add any variables needed by this action to <dict>. - Variable(s) will appear to originate from <source>.""" - raise ParseInternalError('VarAction add_local method must be overriden') - - def write_action(self, vadict, dict2=None, any_scope=False): - """Return a string setting implementing the action of <self>. - Variables must be in <dict> or <dict2>""" - errmsg = 'VarAction write_action method must be overriden' - raise ParseInternalError(errmsg) - - def equiv(self, vmatch): - """Return True iff <vmatch> is equivalent to <self>. - Equivalence at this level is tested by comparing the type - of the objects. - equiv should be overridden with a method that first calls this - method and then tests class-specific object data.""" - return vmatch.__class__ == self.__class__ - - def add_to_list(self, vlist): - """Add <self> to <vlist> unless <self> or its equivalent is - already in <vlist>. This method should not need to be overriden. - Return the (possibly modified) list""" - ok_to_add = True - for vlist_action in vlist: - if vlist_action.equiv(self): - ok_to_add = False - break - # end if - # end for - if ok_to_add: - vlist.append(self) - # end if - return vlist - -############################################################################### - -class VarLoopSubst(VarAction): - """A class to handle required loop substitutions where the host model - (or a suite part) does not provide a loop-like variable used by a - suite part or scheme or where a host model passes a subset of a - dimension at run time.""" - - def __init__(self, missing_stdname, required_stdnames, - local_name, set_action): - """Initialize this variable loop substitution""" - self._missing_stdname = missing_stdname - self._local_name = local_name - if isinstance(required_stdnames, Var): - self._required_stdnames = (required_stdnames,) - else: - # Make sure required_stdnames is iterable - try: - _ = (v for v in required_stdnames) - self._required_stdnames = required_stdnames - except TypeError: - emsg = "required_stdnames must be a tuple or a list" - raise ParseInternalError(emsg) - # end try - # end if - self._set_action = set_action - super().__init__() - - def has_subst(self, vadict, any_scope=False): - """Determine if variables for the required standard names of this - VarLoopSubst object are present in <vadict> (or in the parents of - <vadict>) if <any_scope> is True. - Return a list of the required variables on success, None on failure. - """ - # A template for 'missing' should be in the standard variable list - subst_list = list() - for name in self.required_stdnames: - svar = vadict.find_variable(standard_name=name, any_scope=any_scope) - if svar is None: - subst_list = None - break - # end i - subst_list.append(svar) - # end for - return subst_list - - def add_local(self, vadict, source, run_env): - """Add a Var created from the missing name to <vadict>""" - if self.missing_stdname not in vadict: - lname = self._local_name - local_name = vadict.new_internal_variable_name(prefix=lname) - prop_dict = {'standard_name':self.missing_stdname, - 'local_name':local_name, - 'type':'integer', 'units':'count', 'dimensions':'()'} - var = Var(prop_dict, source, run_env) - vadict.add_variable(var, run_env, exists_ok=True, gen_unique=True) - # end if - - def equiv(self, vmatch): - """Return True iff <vmatch> is equivalent to <self>. - Equivalence is determined by matching the missing standard name - and the required standard names""" - is_equiv = super().equiv(vmatch) - if is_equiv: - is_equiv = vmatch.missing_stdname == self.missing_stdname - # end if - if is_equiv: - for dim1, dim2 in zip(vmatch.required_stdnames, - self.required_stdnames): - if dim1 != dim2: - is_equiv = False - break - # end if - # end for - # end if - return is_equiv - - def write_action(self, vadict, dict2=None, any_scope=False): - """Return a string setting the correct values for our - replacement variable. Variables must be in <vadict> or <dict2>""" - action_dict = {} - if self._set_action: - for stdname in self.required_stdnames: - var = vadict.find_variable(standard_name=stdname, - any_scope=any_scope) - if (var is None) and (dict2 is not None): - var = dict2.find_variable(standard_name=stdname, - any_scope=any_scope) - # end if - if var is None: - errmsg = "Required variable, {}, not found" - raise CCPPError(errmsg.format(stdname)) - # end if - action_dict[stdname] = var.get_prop_value('local_name') - # end for - var = vadict.find_variable(standard_name=self.missing_stdname) - if var is None: - errmsg = "Required variable, {}, not found" - raise CCPPError(errmsg.format(self.missing_stdname)) - # end if - action_dict[self.missing_stdname] = var.get_prop_value('local_name') - # end if - return self._set_action.format(**action_dict) - - def write_metadata(self, mfile): - """Write our properties as metadata to <mfile>""" - pass # Currently no properties to write - - @property - def required_stdnames(self): - """Return the _required_stdnames for this object""" - return self._required_stdnames - - @property - def missing_stdname(self): - """Return the _missing_stdname for this object""" - return self._missing_stdname - - def __repr__(self): - """Return string representing this VarLoopSubst object""" - action_dict = {} - repr_str = '' - if self._set_action: - for stdname in self.required_stdnames: - action_dict[stdname] = stdname - # end for - action_dict[self.missing_stdname] = self.missing_stdname - repr_str = self._set_action.format(**action_dict) - else: - repr_str = "{} => {}".format(self.missing_stdname, - ':'.join(self.required_stdnames)) - # end if - return repr_str - - def __str__(self): - """Return print string for this VarLoopSubst object""" - return "<{}>".format(self.__repr__()) - -# Substitutions where a new variable must be created -CCPP_VAR_LOOP_SUBSTS = { - 'horizontal_loop_extent' : - VarLoopSubst('horizontal_loop_extent', - ('horizontal_loop_begin', 'horizontal_loop_end'), 'ncol', - '{} = {} - {} + 1'.format('{horizontal_loop_extent}', - '{horizontal_loop_end}', - '{horizontal_loop_begin}')), - 'horizontal_loop_begin' : - VarLoopSubst('horizontal_loop_begin', - ('ccpp_constant_one',), 'one', '{horizontal_loop_begin} = 1'), - 'horizontal_loop_end' : - VarLoopSubst('horizontal_loop_end', - ('horizontal_loop_extent',), 'ncol', - '{} = {}'.format('{horizontal_loop_end}', - '{horizontal_loop_extent}')), - 'vertical_layer_dimension' : - VarLoopSubst('vertical_layer_dimension', - ('vertical_layer_index',), 'layer_index', ''), - 'vertical_interface_dimension' : - VarLoopSubst('vertical_interface_dimension', - ('vertical_interface_index',), 'level_index', '') -} - -############################################################################### - -class VarDictionary(OrderedDict): - """ - A class to store and cross-check variables from one or more metadata - headers. The class also serves as a scoping construct so that a variable - can be found in an innermost available scope. - The dictionary is organized by standard_name. It is an error to try - to add a variable if its standard name is already in the dictionary. - Scoping is a tree of VarDictionary objects. - >>> VarDictionary('foo', _MVAR_DUMMY_RUN_ENV) - VarDictionary(foo) - >>> VarDictionary('bar', _MVAR_DUMMY_RUN_ENV, variables={}) - VarDictionary(bar) - >>> test_dict = VarDictionary('baz', _MVAR_DUMMY_RUN_ENV, variables=Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'scheme', ParseContext()), _MVAR_DUMMY_RUN_ENV)) - >>> print(test_dict.name) - baz - >>> print(test_dict.variable_list()) #doctest: +ELLIPSIS - [<metavar.Var hi_mom: foo at 0x...>] - >>> print("{}".format(VarDictionary('baz', _MVAR_DUMMY_RUN_ENV, variables=Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'scheme', ParseContext()), _MVAR_DUMMY_RUN_ENV)))) - VarDictionary(baz, ['hi_mom']) - >>> test_dict = VarDictionary('qux', _MVAR_DUMMY_RUN_ENV, variables=[Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'scheme', ParseContext()), _MVAR_DUMMY_RUN_ENV)]) - >>> print(test_dict.name) - qux - >>> print(test_dict.variable_list()) #doctest: +ELLIPSIS - [<metavar.Var hi_mom: foo at 0x...>] - >>> VarDictionary('boo', _MVAR_DUMMY_RUN_ENV).add_variable(Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'scheme', ParseContext()), _MVAR_DUMMY_RUN_ENV), _MVAR_DUMMY_RUN_ENV) - - >>> VarDictionary('who', _MVAR_DUMMY_RUN_ENV, variables=[Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'scheme', ParseContext()), _MVAR_DUMMY_RUN_ENV)]).prop_list('local_name') - ['foo'] - >>> VarDictionary('who', _MVAR_DUMMY_RUN_ENV, variables=[Var({'local_name' : 'who_var1', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'scheme', ParseContext()), _MVAR_DUMMY_RUN_ENV),Var({'local_name' : 'who_var', 'standard_name' : 'bye_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'scheme', ParseContext()), _MVAR_DUMMY_RUN_ENV)]).new_internal_variable_name() - 'who_var2' - >>> VarDictionary('who', _MVAR_DUMMY_RUN_ENV, variables=[Var({'local_name' : 'who_var1', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'scheme', ParseContext()), _MVAR_DUMMY_RUN_ENV)]).new_internal_variable_name(prefix='bar') - 'bar' - >>> VarDictionary('glitch', _MVAR_DUMMY_RUN_ENV, variables=Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'scheme', ParseContext()), _MVAR_DUMMY_RUN_ENV)).add_variable(Var({'local_name' : 'bar', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname2', 'DDT', ParseContext()), _MVAR_DUMMY_RUN_ENV), _MVAR_DUMMY_RUN_ENV) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Invalid Duplicate standard name, 'hi_mom', at <standard input>: - """ - - def __init__(self, name, run_env, variables=None, - parent_dict=None): - """Unlike dict, VarDictionary only takes a Var or Var list""" - super().__init__() - self.__name = name - self.__run_env = run_env - self.__parent_dict = parent_dict - if parent_dict is not None: - parent_dict.add_sub_scope(self) - # end if - self.__sub_dicts = list() - self.__local_names = {} # local names in use - if isinstance(variables, Var): - self.add_variable(variables, run_env) - elif isinstance(variables, list): - for var in variables: - self.add_variable(var, run_env) - # end for - elif isinstance(variables, VarDictionary): - for stdname in variables.keys(): - self[stdname] = variables[stdname] - # end for - elif isinstance(variables, dict): - # variables may not be in 'order', but we accept them anyway - for key in variables.keys(): - var = variables[key] - stdname = var.get_prop_value('standard_name') - self[stdname] = variables[key] - # end for - elif variables is not None: - raise ParseInternalError(f'Illegal type for variables, {type_name(variables)} in {self.name}') - # end if - - @property - def name(self): - """Return this dictionary's name""" - return self.__name - - @property - def parent(self): - """Return the parent dictionary of this dictionary""" - return self.__parent_dict - - @staticmethod - def include_var_in_list(var, std_vars, loop_vars, consts): - """Return True iff <var> is of a type allowed by the logicals, - <std_vars> (not constants or loop_vars), - <loop_vars> a variable ending in '_extent', '_begin', '_end', or - <consts> a variable with the 'protected' property. - """ - standard_name = var.get_prop_value('standard_name') - const_var = standard_name in CCPP_CONSTANT_VARS - loop_var = standard_name in CCPP_LOOP_VAR_STDNAMES - include_var = (consts and const_var) or (loop_var and loop_vars) - if not include_var: - std_var = not (loop_var or const_var) - include_var = std_vars and std_var - # end if - return include_var - - def variable_list(self, recursive=False, - std_vars=True, loop_vars=True, consts=True): - """Return a list of all variables""" - if recursive and (self.__parent_dict is not None): - vlist = self.__parent_dict.variable_list(recursive=recursive, - std_vars=std_vars, - loop_vars=loop_vars, - consts=consts) - else: - vlist = list() - # end if - for stdnam in self: - var = self[stdnam] - if self.include_var_in_list(var, std_vars=std_vars, - loop_vars=loop_vars, consts=consts): - vlist.append(var) - # end if - # end for - return vlist - - def add_variable(self, newvar, run_env, exists_ok=False, gen_unique=False, - adjust_intent=False): - """Add <newvar> if it does not conflict with existing entries - If <exists_ok> is True, attempting to add an identical copy is okay. - If <gen_unique> is True, a new local_name will be created if a - local_name collision is detected. - if <adjust_intent> is True, adjust conflicting intents to inout.""" - standard_name = newvar.get_prop_value('standard_name') - cvar = self.find_variable(standard_name=standard_name, any_scope=False) - if (standard_name in self) and (not exists_ok): - # We already have a matching variable, error! - if self.__run_env.logger is not None: - emsg = "Attempt to add duplicate variable, {} from {}" - self.__run_env.logger.error(emsg.format(standard_name, - newvar.source.name)) - # end if - emsg = "(duplicate) standard name in {}" - if cvar is not None: - emsg += ", defined at {}".format(cvar.context) - # end if - raise ParseSyntaxError(emsg.format(self.name), - token=standard_name, context=newvar.context) - # end if - if cvar is not None: - compat = cvar.compatible(newvar, run_env) - if compat.compat: - # Check for intent mismatch - vintent = cvar.get_prop_value('intent') - dintent = newvar.get_prop_value('intent') - # XXgoldyXX: Add special case for host variables here? - if vintent != dintent: - if adjust_intent: - if (vintent == 'in') and (dintent in ['inout', 'out']): - cvar.adjust_intent(newvar) - elif ((vintent == 'out') and - (dintent in ['inout', 'in'])): - cvar.adjust_intent(newvar) - # No else, variables are compatible - else: - emsg = "Attempt to add incompatible variable to {}" - emsg += "\nintent mismatch: {} ({}){} != {} ({}){}" - nlname = newvar.get_prop_value('local_name') - clname = cvar.get_prop_value('local_name') - nctx = context_string(newvar.context) - cctx = context_string(cvar.context) - raise CCPPError(emsg.format(self.name, - clname, vintent, cctx, - nlname, dintent, nctx)) - # end if - # end if - else: - if self.__run_env.logger is not None: - emsg = "Attempt to add incompatible variable, {} from {}" - emsg += "\n{}".format(compat.incompat_reason) - self.__run_env.logger.error(emsg.format(standard_name, - newvar.source.name)) - # end if - nlname = newvar.get_prop_value('local_name') - clname = cvar.get_prop_value('local_name') - cstr = context_string(cvar.context, with_comma=True) - errstr = "new variable, {}, incompatible {} between {}{} and" - raise ParseSyntaxError(errstr.format(nlname, - compat.incompat_reason, - clname, cstr), - token=standard_name, - context=newvar.context) - # end if - # end if - # Check if local_name exists in Group. If applicable, Create new - # variable with unique name. There are two instances when new names are - # created: - # - Same <local_name> used in different DDTs. - # - Different <standard_name> using the same <local_name> in a Group. - # During the Group analyze phase, <gen_unique> is True. - lname = newvar.get_prop_value('local_name') - lvar = self.find_local_name(lname) - if lvar is not None: - # Check if <lvar> is part of a different DDT than <newvar>. - # The API uses the full variable references when calling the Group Caps, - # <lvar.call_string(self))> and <newvar.call_string(self)>. - # Within the context of a full reference, it is allowable for local_names - # to be the same in different data containers. - newvar_callstr = newvar.call_string(self) - lvar_callstr = lvar.call_string(self) - if newvar_callstr and lvar_callstr: - if newvar_callstr != lvar_callstr: - if not gen_unique: - exists_ok = True - # end if - # end if - # end if - if gen_unique: - new_lname = self.new_internal_variable_name(prefix=lname) - newvar = newvar.clone(new_lname) - # Local_name needs to be the local_name for the new - # internal variable, otherwise multiple instances of the same - # local_name in the Group cap will all be overwritten with the - # same local_name - lname = new_lname - elif not exists_ok: - errstr = f"Invalid local_name: {lname} already registered" - raise ParseSyntaxError(errstr, context=newvar.source.context) - # end if (no else, things are okay) - # end if (no else, things are okay) - # Check if this variable has a parent (i.e., it is an array reference) - aref = newvar.array_ref(local_name=lname) - if aref is not None: - pname = aref.group(1).strip() - pvar = self.find_local_name(pname) - if pvar is not None: - newvar.parent = pvar - # end if - # end if - # If we make it to here without an exception, add the variable - if standard_name not in self: - self[standard_name] = newvar - # end if - lname = lname.lower() - if lname not in self.__local_names: - self.__local_names[lname] = standard_name - # end if - - def remove_variable(self, standard_name): - """Remove <standard_name> from the dictionary. - Ignore if <standard_name> is not in dict - """ - if standard_name in self: - del self[standard_name] - # end if - - def add_variable_dimensions(self, var, ignore_sources, suite_type, - to_dict=None, adjust_intent=False): - """Attempt to find a source for each dimension in <var> and add that - Variable to this dictionary or to <to_dict>, if passed. - Dimension variables which are found but whose Source is in - <ignore_sources> are not added to this dictionary. - Dimension variabes which are found at the suite level (determined - by <suite_type>) are also not added to this dictionary because - module-level suite variables are accessible by any phase. - Return an error string on failure.""" - - err_ret = '' - ctx = '' - vdims = var.get_dim_stdnames(include_constants=False) - for dimname in vdims: - if to_dict: - present = to_dict.find_variable(standard_name=dimname, - any_scope=False) - else: - present = None - # end if - if not present: - present = self.find_variable(standard_name=dimname, - any_scope=False) - # end if - if not present: - dvar = self.find_variable(standard_name=dimname, any_scope=True) - if dvar and dvar.source.ptype == suite_type: - # Do nothing - this is a module-level variable so we don't - # need to add it to any dictionaries - return - # end if - if dvar and (dvar.source.ptype not in ignore_sources): - if to_dict: - to_dict.add_variable(dvar, self.__run_env, - exists_ok=True, - adjust_intent=adjust_intent) - else: - self.add_variable(dvar, self.__run_env, exists_ok=True, - adjust_intent=adjust_intent) - # end if - else: - if err_ret: - err_ret += '\n' - else: - ctx = context_string(var.context) - # end if - vstdname = var.get_prop_value('standard_name') - err_ret += f"{self.name}: " - err_ret += f"Cannot find variable for dimension, {dimname}, of {vstdname}{ctx}" - if dvar: - lname = dvar.get_prop_value('local_name') - dctx = context_string(dvar.context) - err_ret += f"\nFound {lname} from excluded source, '{dvar.source.ptype}'{dctx}" - # end if - # end if - # end if - # end for - return err_ret - - def find_variable(self, standard_name=None, source_var=None, - any_scope=True, clone=None, - search_call_list=False, loop_subst=False): - """Attempt to return the variable matching <standard_name>. - if <standard_name> is None, the standard name from <source_var> is used. - It is an error to pass both <standard_name> and <source_var> if - the standard name of <source_var> is not the same as <standard_name>. - If <any_scope> is True, search parent scopes if not in current scope. - If the variable is not found and <clone> is not None, add a clone of - <clone> to this dictionary. - If the variable is not found and <clone> is None, return None. - <search_call_list> and <loop_subst> are not used in this base class - but are included to provide a consistent interface. - """ - if standard_name is None: - if source_var is None: - emsg = "One of <standard_name> or <source_var> must be passed." - raise ParseInternalError(emsg) - # end if - standard_name = source_var.get_prop_value('standard_name') - elif source_var is not None: - stest = source_var.get_prop_value('standard_name') - if stest != standard_name: - emsg = ("<standard_name> and <source_var> must match " + - "if both are passed.") - raise ParseInternalError(emsg) - # end if - # end if - if standard_name in CCPP_CONSTANT_VARS: - var = CCPP_CONSTANT_VARS[standard_name] - elif standard_name in self: - var = self[standard_name] - elif any_scope and (self.__parent_dict is not None): - src_clist = search_call_list - var = self.__parent_dict.find_variable(standard_name=standard_name, - source_var=source_var, - any_scope=any_scope, - clone=clone, - search_call_list=src_clist, - loop_subst=loop_subst) - else: - var = None - # end if - if (var is None) and (clone is not None): - lname = clone.get_prop_value['local_name'] - new_name = self.new_internal_variable_name(prefix=lname) - var = clone.clone(new_name) - # end if - return var - - def find_local_name(self, local_name, any_scope=False): - """Return a variable in this dictionary with local_name = <local_name> - or return None if no such variable is currently in the dictionary""" - pvar = None - lname = local_name.lower() # Case is insensitive for local names - if lname in self.__local_names: - stdname = self.__local_names[lname] - pvar = self.find_variable(standard_name=stdname, any_scope=False) - if not pvar: - emsg = 'VarDictionary {} should have standard_name, {}, ' - emsg += 'based on local_name {}' - raise ParseInternalError(emsg.format(self.name, - stdname, local_name)) - # end if (no else, pvar is fine) - elif any_scope and (self.__parent_dict is not None): - pvar = self.__parent_dict.find_local_name(local_name, - any_scope=any_scope) - # end if - return pvar - - def find_error_variables(self, any_scope=False, clone_as_out=False): - """Find and return a consistent set of error variables in this - dictionary. - First, attempt to find the set of errflg and errmsg. - Currently, there is no alternative but it will be inserted here. - If a consistent set is not found, return an empty list. - """ - err_vars = list() - # Look for the combo of errflg and errmsg - errflg = self.find_variable(standard_name="ccpp_error_code", - any_scope=any_scope) - errmsg = self.find_variable(standard_name="ccpp_error_message", - any_scope=any_scope) - if (errflg is not None) and (errmsg is not None): - if clone_as_out: - eout = errmsg.get_prop_value('intent') - if eout != 'out': - subst_dict = {'intent':'out'} - errmsg = errmsg.clone(subst_dict) - # end if - # end if - err_vars.append(errmsg) - if clone_as_out: - eout = errflg.get_prop_value('intent') - if eout != 'out': - subst_dict = {'intent':'out'} - errflg = errflg.clone(subst_dict) - # end if - # end if - err_vars.append(errflg) - # end if - return err_vars - - def add_sub_scope(self, sub_dict): - """Add a child dictionary to enable traversal""" - self.__sub_dicts.append(sub_dict) - - def sub_dictionaries(self): - """Return a list of this dictionary's sub-dictionaries""" - return list(self.__sub_dicts) - - def prop_list(self, prop_name, std_vars=True, loop_vars=True, consts=True): - """Return a list of the <prop_name> property for each variable. - std_vars are variables which are neither constants nor loop variables. - """ - plist = list() - for var in self.values(): - if self.include_var_in_list(var, std_vars=std_vars, - loop_vars=loop_vars, consts=consts): - plist.append(var.get_prop_value(prop_name)) - # end if - # end for - return plist - - def declare_variables(self, outfile, indent, dummy=False, - std_vars=True, loop_vars=True, consts=True): - """Write out the declarations for this dictionary's variables""" - for standard_name in self.keys(): - var = self.find_variable(standard_name=standard_name, - any_scope=False) - if self.include_var_in_list(var, std_vars=std_vars, - loop_vars=loop_vars, consts=consts): - self[standard_name].write_def(outfile, indent, self, - dummy=dummy) - # end if - # end for - - def merge(self, other_dict, run_env): - """Add new entries from <other_dict>""" - for ovar in other_dict.variable_list(): - self.add_variable(ovar, run_env) - # end for - - @staticmethod - def loop_var_okay(standard_name, is_run_phase): - """If <standard_name> is a loop variable, return True only if it - is appropriate for the phase (e.g., horizontal_loop_extent is okay - during a run phase only while horizontal_dimension is not allowed - during a run phase). - If <standard_name> is not a loop variable, return True""" - if (standard_name in CCPP_LOOP_VAR_STDNAMES) and (not is_run_phase): - # Prohibit looking for loop variables except in run phases - retval = False - elif (standard_name == "horizontal_dimension") and is_run_phase: - # horizontal_dimension should not be used in run phase - retval = False - else: - retval = True - # end if - return retval - - def __str__(self): - """Return a string that represents this dictionary object""" - return f"VarDictionary({self.name}, {list(self.keys())})" - - def __repr__(self): - """Return an unique representation for this object""" - srepr = super().__repr__() - vstart = len("VarDictionary") + 1 - if len(srepr) > vstart + 1: - comma = ", " - else: - comma = "" - # end if - return f"VarDictionary({self.name}{comma}{srepr[vstart:]}" - - def __del__(self): - """Attempt to delete all of the variables in this dictionary""" - self.clear() - - def __eq__(self, other): - """Override == to restore object equality, not dictionary - list equality""" - return self is other - - @classmethod - def loop_var_match(cls, standard_name): - """Return a VarLoopSubst if <standard_name> is a loop variable, - otherwise, return None""" - # Strip off 'ccpp_constant_one:', if present - if standard_name[0:18] == 'ccpp_constant_one:': - beg = 18 - else: - beg = 0 - # end if - if standard_name[beg:] in CCPP_VAR_LOOP_SUBSTS: - vmatch = CCPP_VAR_LOOP_SUBSTS[standard_name[beg:]] - else: - vmatch = None - # end if - return vmatch - - def find_loop_dim_match(self, dim_string): - """Find a match in local dict for <dim_string>. That is, if - <dim_string> has a loop dim substitution, and each standard name - in that substitution is in self, return the equivalent local - name string.""" - ldim_string = None - if dim_string in CCPP_LOOP_DIM_SUBSTS: - lnames = list() - std_subst = CCPP_LOOP_DIM_SUBSTS[dim_string].split(':') - for ssubst in std_subst: - svar = self.find_variable(standard_name=ssubst, any_scope=False) - if svar is not None: - lnames.append(svar.call_string(self)) - else: - break - # end if - # end for - if len(lnames) == len(std_subst): - ldim_string = ':'.join(lnames) - # end if - # end if - return ldim_string - - @classmethod - def find_loop_dim_from_index(cls, index_string): - """Given a loop index standard name, find the related loop dimension. - """ - loop_dim_string = None - for dim_string in CCPP_LOOP_DIM_SUBSTS: - if index_string == CCPP_LOOP_DIM_SUBSTS[dim_string]: - loop_dim_string = dim_string - break - # end if - # end for - return loop_dim_string - - def find_loop_subst(self, standard_name, any_scope=True, context=None): - """If <standard_name> is of the form <standard_name>_extent and that - variable is not in the dictionary, substitute a tuple of variables, - (<standard_name>_begin, <standard_name>_end), if those variables are - in the dictionary. - If <standard_name>_extent *is* present, return that variable as a - range, ('ccpp_constant_one', <standard_name>_extent) - In other cases, return None - """ - loop_var = VarDictionary.loop_var_match(standard_name) - logger_str = None - if loop_var is not None: - # Let us see if we can fix a loop variable - dict_var = self.find_variable(standard_name=standard_name, - any_scope=any_scope) - if dict_var is not None: - var_one = CCPP_CONSTANT_VARS['ccpp_constant_one'] - my_var = (var_one, dict_var) - if self.__run_env.logger is not None: - lstr = "loop_subst: found {}{}" - logger_str = lstr.format(standard_name, - context_string(context)) - # end if - else: - my_vars = [self.find_variable(standard_name=x, - any_scope=any_scope) - for x in loop_var] - if None not in my_vars: - my_var = tuple(my_vars) - if self.__run_env.logger is not None: - names = [x.get_prop_value('local_name') - for x in my_vars] - lstr = "loop_subst: {} ==> ({}){}" - logger_str = lstr.format(standard_name, - ', '.join(names), - context_string(context)) - # end if - else: - if self.__run_env.logger is not None: - lstr = "loop_subst: {} ==> (??) FAILED{}" - logger_str = lstr.format(standard_name, - context_string(context)) - # end if - my_var = None - # end if - # end if - else: - if self.__run_env.logger is not None: - lstr = "loop_subst: {} is not a loop variable{}" - logger_str = lstr.format(standard_name, - context_string(context)) - # end if - my_var = None - # end if - if logger_str is not None: - self.__run_env.logger.debug(logger_str) - # end if - return my_var - - def var_call_string(self, var, loop_vars=None): - """Construct the actual argument string for <var> by translating - standard names to local names. String includes array bounds. - if <loop_vars> is present, look there first for array bounds, - even if usage requires a loop substitution. - """ - return var.call_string(self, loop_vars=loop_vars) - - def new_internal_variable_name(self, prefix=None, max_len=63): - """Find a new local variable name for this dictionary. - The new name begins with <prefix>_<self.name> or with <self.name> - (where <self.name> is this VarDictionary's name) if <prefix> is None. - The new variable name is kept to a maximum length of <max_len>. - """ - index = 0 - if prefix is None: - var_prefix = '{}_var'.format(self.name) - else: - var_prefix = '{}'.format(prefix) - # end if - varlist = [x for x in self.__local_names.keys() if var_prefix in x] - newvar = None - while newvar is None: - if index == 0: - newvar = var_prefix - else: - newvar = '{}{}'.format(var_prefix, index) - # end if - index = index + 1 - if len(newvar) > max_len: - var_prefix = var_prefix[:-1] - newvar = None - elif newvar in varlist: - newvar = None - # end if - # end while - return newvar - -############################################################################### - -# List of constant variables which are universally available -CCPP_CONSTANT_VARS = \ - VarDictionary('CCPP_CONSTANT_VARS', _MVAR_DUMMY_RUN_ENV, - variables=[ccpp_standard_var('ccpp_constant_one', 'module', - _MVAR_DUMMY_RUN_ENV)]) - -############################################################################### diff --git a/scripts/mkcap.py b/scripts/mkcap.py deleted file mode 100755 index c5e88362..00000000 --- a/scripts/mkcap.py +++ /dev/null @@ -1,831 +0,0 @@ -#!/usr/bin/env python3 -# -# Script to generate a cap module and subroutines -# from a scheme xml file. -# - -import copy -import logging -import os -import sys -import getopt -import xml.etree.ElementTree as ET - -from common import CCPP_INTERNAL_VARIABLES -from common import STANDARD_VARIABLE_TYPES, STANDARD_CHARACTER_TYPE -from common import isstring, string_to_python_identifier -from conversion_tools import unit_conversion - -############################################################################### - -class Var(object): - - def __init__(self, **kwargs): - self._standard_name = None - self._long_name = None - self._units = None - self._local_name = None - self._type = None - self._dimensions = [] - self._container = None - self._kind = None - self._intent = None - self._active = None - self._optional = None - self._pointer = False - self._target = None - self._actions = { 'in' : None, 'out' : None } - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - @property - def standard_name(self): - '''Get the name of the variable.''' - return self._standard_name - - @standard_name.setter - def standard_name(self, value): - self._standard_name = value - - @property - def long_name(self): - '''Get the name of the variable.''' - return self._long_name - - @long_name.setter - def long_name(self, value): - self._long_name = value - - @property - def units(self): - '''Get the units of the variable.''' - return self._units - - @units.setter - def units(self, value): - self._units = value - - @property - def local_name(self): - '''Get the local variable name of the variable.''' - return self._local_name - - @local_name.setter - def local_name(self, value): - self._local_name = value - - @property - def type(self): - '''Get the type of the variable.''' - return self._type - - @type.setter - def type(self, value): - self._type = value - - @property - def dimensions(self): - '''Get the dimensions of the variable.''' - return self._dimensions - - @dimensions.setter - def dimensions(self, value): - if not isinstance(value, list): - raise TypeError('Invalid type for variable property dimensions, must be a list') - self._dimensions = value - - @property - def rank(self): - '''Get the rank of the variable. Originally, this was an integer indicating - the number of dimensions (therefore the name), now it is a list of colons to use - for assumed-size array definitions in Fortran.''' - if len(self._dimensions) == 0: - return '' - else: - return '('+ ','.join([':'] * len(self._dimensions)) +')' - - @property - def kind(self): - '''Get the kind of the variable.''' - return self._kind - - @kind.setter - def kind(self, value): - self._kind = value - - @property - def intent(self): - '''Get the intent of the variable.''' - return self._intent - - @intent.setter - def intent(self, value): - if not value in ['none', 'in', 'out', 'inout']: - raise ValueError('Invalid value {0} for variable property intent'.format(value)) - self._intent = value - - @property - def active(self): - '''Get the active attribute of the variable.''' - return self._active - - @active.setter - def active(self, value): - if not isinstance(value, str): - raise ValueError('Invalid value {0} for variable property active, must be a string'.format(value)) - self._active = value - - @property - def optional(self): - '''Get the optional attribute of the variable.''' - return self._optional - - @optional.setter - def optional(self, value): - if not isinstance(value, str): - raise ValueError('Invalid value {0} for variable property optional, must be a string'.format(value)) - self._optional = value - - # Pointer is not set by parsing metadata attributes, but by mkstatic. - # This is a quick and dirty solution! - @property - def pointer(self): - '''Get the pointer attribute of the variable.''' - return self._pointer - - @pointer.setter - def pointer(self, value): - if not isinstance(value, bool): - raise ValueError('Invalid value {0} for variable property pointer, must be a logical'.format(value)) - self._pointer = value - - @property - def target(self): - '''Get the target of the variable.''' - return self._target - - @target.setter - def target(self, value): - self._target = value - - @property - def container(self): - '''Get the container of the variable.''' - return self._container - - @container.setter - def container(self, value): - self._container = value - - @property - def actions(self): - '''Get the action strings for the variable.''' - return self._actions - - @actions.setter - def actions(self, values): - if isinstance(values, dict): - for key in values.keys(): - if key in ['in', 'out'] and isstring(values[key]): - self._actions[key] = values[key] - else: - raise Exception('Invalid values for variable attribute actions.') - else: - raise Exception('Invalid values for variable attribute actions.') - - def compatible(self, other): - """Test if the variable is compatible another variable. This requires - that certain variable attributes are identical. Others, for example - len=... for character variables have less strict requirements: accept - character(len=*) as compatible with character(len=INTEGER_VALUE). - We defer testing units here and catch incompatible units later when - unit-conversion code is autogenerated.""" - if self.type == 'character': - if (self.kind == 'len=*' and other.kind.startswith('len=')) or \ - (self.kind.startswith('len=') and other.kind == 'len=*'): - return self.standard_name == other.standard_name \ - and self.type == other.type \ - and self.rank == other.rank - return self.standard_name == other.standard_name \ - and self.type == other.type \ - and self.kind == other.kind \ - and self.rank == other.rank - - def convert_to(self, units): - """Generate action to convert data in the variable's units to other units""" - function_name = '{0}__to__{1}'.format(string_to_python_identifier(self.units), string_to_python_identifier(units)) - try: - function = getattr(unit_conversion, function_name) - logging.info('Automatic unit conversion from {0} to {1} for {2} after returning from {3}'.format(self.units, units, self.standard_name, self.container)) - except AttributeError: - raise Exception('Error, automatic unit conversion from {0} to {1} for {2} in {3} not implemented'.format(self.units, units, self.standard_name, self.container)) - conversion = function() - self._actions['out'] = function() - - def convert_from(self, units): - """Generate action to convert data in other units to the variable's units""" - function_name = '{1}__to__{0}'.format(string_to_python_identifier(self.units), string_to_python_identifier(units)) - try: - function = getattr(unit_conversion, function_name) - logging.info('Automatic unit conversion from {0} to {1} for {2} before entering {3}'.format(self.units, units, self.standard_name, self.container)) - except AttributeError: - raise Exception('Error, automatic unit conversion from {1} to {0} for {2} in {3} not implemented'.format(self.units, units, self.standard_name, self.container)) - conversion = function() - self._actions['in'] = function() - - def dimstring_local_names(self, metadata, assume_shape = False): - '''Create the dimension string for assumed shape or explicit arrays - in Fortran. Requires a metadata dictionary to resolve the dimensions, - which are in CCPP standard names, to local variable names. If the - optional argument assume_shape is True, return an assumed shape - dimension string with the upper bound being left undefined.''' - # Simplest case: scalars - if len(self.dimensions) == 0: - return '' - dimstring = [] - # Arrays - for dim in self.dimensions: - # Handle dimensions like "A:B", "A:3", "-1:Z" - if ':' in dim: - dims = [ x.lower() for x in dim.split(':')] - try: - dim0 = int(dims[0]) - dim0 = dims[0] - except ValueError: - if not dims[0].lower() in metadata.keys(): - raise Exception('Dimension {}, required by variable {}, not defined in metadata'.format( - dims[0].lower(), self.standard_name)) - dim0 = metadata[dims[0].lower()][0].local_name - try: - dim1 = int(dims[1]) - dim1 = dims[1] - except ValueError: - if not dims[1].lower() in metadata.keys(): - raise Exception('Dimension {}, required by variable {}, not defined in metadata'.format( - dims[1].lower(), self.standard_name)) - dim1 = metadata[dims[1].lower()][0].local_name - # Single dimensions - else: - dim0 = 1 - try: - dim1 = int(dim) - dim1 = dim - except ValueError: - if not dim.lower() in metadata.keys(): - raise Exception('Dimension {}, required by variable {}, not defined in metadata'.format( - dim.lower(), self.standard_name)) - dim1 = metadata[dim.lower()][0].local_name - if assume_shape: - dimstring.append('{}:'.format(dim0)) - else: - dimstring.append('{}:{}'.format(dim0, dim1)) - return '({})'.format(','.join(dimstring)) - - def print_module_use(self): - '''Print the module use line for the variable.''' - for item in self.container.split(' '): - if item.startswith('MODULE_'): - module = item.replace('MODULE_', '') - break - str = 'use {module}, only: {varname}'.format(module=module,varname=self.local_name) - return str - - def print_def_intent(self, metadata): - '''Print the definition line for the variable, using intent. Use the metadata - dictionary to resolve lower bounds for array dimensions.''' - # Resolve dimensisons to local names using undefined upper bounds (assumed shape) - dimstring = self.dimstring_local_names(metadata, assume_shape = True) - # It is an error for host model variables to have the optional attribute in the metadata - if self.optional == 'T': - error_message = "This routine should only be called for host model variables" + \ - " that cannot have the optional metadata attribute, but got self.optional=T" - raise Exception(error_message) - # If the host variable is potentially unallocated, add optional and target to variable declaration - elif not self.active == 'T': - optional = ', optional, target' - else: - # Always declare as target variable so that locally defined pointers can point to it - optional = ', target' - # - if self.type in STANDARD_VARIABLE_TYPES: - if self.kind: - str = "{s.type}({s._kind}), intent({s.intent}){optional} :: {s.local_name}{dimstring}" - else: - str = "{s.type}, intent({s.intent}){optional} :: {s.local_name}{dimstring}" - else: - if self.kind: - error_message = "Generating variable definition statements for derived types with" + \ - " kind attributes not implemented; variable: {0}".format(self.standard_name) - raise Exception(error_message) - else: - str = "type({s.type}), intent({s.intent}){optional} :: {s.local_name}{dimstring}" - return str.format(s=self, optional=optional, dimstring=dimstring) - - def print_def_local(self, metadata): - '''Print the definition line for the variable, assuming it is a local variable.''' - # It is an error for local variables to have the active attribute - if not self.active == 'T': - error_message = "This routine should only be called for local variables" + \ - " that cannot have an active attribute other than the" +\ - " default T, but got self.active=T" - raise Exception(error_message) - - # If it is a pointer, everything is different! - if self.pointer: - if self.type in STANDARD_VARIABLE_TYPES: - if self.kind: - if self.rank: - str = "{s.type}({s._kind}), dimension{s.rank}, pointer :: p => null()" - else: - str = "{s.type}({s._kind}), pointer :: p => null()" - else: - if self.rank: - str = "{s.type}, dimension{s.rank}, pointer :: p => null()" - else: - str = "{s.type}, pointer :: p => null()" - else: - if self.kind: - error_message = "Generating variable definition statements for derived types with" + \ - " kind attributes not implemented; variable: {0}".format(self.standard_name) - raise Exception(error_message) - else: - if self.rank: - str = "type({s.type}), dimension{s.rank}, pointer :: p => null()" - else: - str = "type({s.type}), pointer :: p => null()" - return str.format(s=self) - else: - # DH* 20241022 WORKAROUND TO ACCOUNT FOR MISSING UPDATES TO CCPP PHYSICS - # W.R.T. DECLARING OPTIONAL VARIABLES IN METADATA AND CODE. ALWAYS USE TARGET - ## If the host variable is potentially unallocated, the active attribute is - ## also set accordingly for the local variable; add target to variable declaration - #if self.optional == 'T': - # target = ', target' - #else: - # target = '' - target = ', target' - # *DH - if self.type in STANDARD_VARIABLE_TYPES: - if self.kind: - if self.rank: - str = "{s.type}({s._kind}), dimension{s.rank}, allocatable{target} :: {s.local_name}" - else: - str = "{s.type}({s._kind}){target} :: {s.local_name}" - else: - if self.rank: - str = "{s.type}, dimension{s.rank}, allocatable{target} :: {s.local_name}" - else: - str = "{s.type}{target} :: {s.local_name}" - else: - if self.kind: - error_message = "Generating variable definition statements for derived types with" + \ - " kind attributes not implemented; variable: {0}".format(self.standard_name) - raise Exception(error_message) - else: - if self.rank: - str = "type({s.type}), dimension{s.rank}, allocatable{target} :: {s.local_name}" - else: - str = "type({s.type}){target} :: {s.local_name}" - return str.format(s=self, target=target) - - def print_debug(self): - '''Print the data retrieval line for the variable.''' - # Scheme variables don't have the active attribute - if 'SCHEME' in self.container: - str='''Contents of {s} (* = mandatory for compatibility): - standard_name = {s.standard_name} * - long_name = {s.long_name} - units = {s.units} * - local_name = {s.local_name} - type = {s.type} * - dimensions = {s.dimensions} - rank = {s.rank} * - kind = {s.kind} * - intent = {s.intent} - optional = {s.optional} - target = {s.target} - container = {s.container} - actions = {s.actions}''' - # Host model variables don't have the optional attribute - else: - str='''Contents of {s} (* = mandatory for compatibility): - standard_name = {s.standard_name} * - long_name = {s.long_name} - units = {s.units} * - local_name = {s.local_name} - type = {s.type} * - dimensions = {s.dimensions} - rank = {s.rank} * - kind = {s.kind} * - intent = {s.intent} - active = {s.active} - target = {s.target} - container = {s.container} - actions = {s.actions}''' - return str.format(s=self) - -class CapsMakefile(object): - - header=''' -# All CCPP caps are defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -CAPS_F90 =''' - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, caps): - if (self.filename is not sys.stdout): - f = open(self.filename, 'w') - else: - f = sys.stdout - - contents = self.header - for cap in caps: - contents += ' \\\n\t {0}'.format(cap) - f.write(contents) - - if (f is not sys.stdout): - f.close() - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - -class CapsCMakefile(object): - - header=''' -# All CCPP caps are defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -set(CAPS -''' - footer=''') -''' - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, caps): - if (self.filename is not sys.stdout): - f = open(self.filename, 'w') - else: - f = sys.stdout - - contents = self.header - for cap in caps: - contents += ' {0}\n'.format(cap) - contents += self.footer - f.write(contents) - - if (f is not sys.stdout): - f.close() - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - -class CapsSourcefile(object): - - header=''' -# All CCPP caps are defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -export CCPP_CAPS="''' - footer='''" -''' - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, caps): - if (self.filename is not sys.stdout): - filepath = os.path.split(self.filename)[0] - if filepath and not os.path.isdir(filepath): - os.makedirs(filepath) - f = open(self.filename, 'w') - else: - f = sys.stdout - - contents = self.header - for cap in caps: - contents += '{0};'.format(cap) - contents = contents.rstrip(';') - contents += self.footer - f.write(contents) - - if (f is not sys.stdout): - f.close() - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - -class SchemesMakefile(object): - - header=''' -# All CCPP schemes are defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -SCHEMES_F = - -SCHEMES_F90 = - -SCHEMES_f = - -SCHEMES_f90 =''' - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, schemes): - if (self.filename is not sys.stdout): - filepath = os.path.split(self.filename)[0] - if filepath and not os.path.isdir(filepath): - os.makedirs(filepath) - f = open(self.filename, 'w') - else: - f = sys.stdout - - contents = self.header - schemes_F = 'SCHEMES_F =' - schemes_F90 = 'SCHEMES_F90 =' - schemes_f = 'SCHEMES_f =' - schemes_f90 = 'SCHEMES_f90 =' - for scheme in schemes: - if scheme.endswith('.F'): - schemes_F += ' \\\n\t {0}'.format(scheme) - elif scheme.endswith('.F90'): - schemes_F90 += ' \\\n\t {0}'.format(scheme) - elif scheme.endswith('.f'): - schemes_f += ' \\\n\t {0}'.format(scheme) - elif scheme.endswith('.f90'): - schemes_f90 += ' \\\n\t {0}'.format(scheme) - contents = contents.replace('SCHEMES_F =', schemes_F) - contents = contents.replace('SCHEMES_F90 =', schemes_F90) - contents = contents.replace('SCHEMES_f =', schemes_f) - contents = contents.replace('SCHEMES_f90 =', schemes_f90) - f.write(contents) - - if (f is not sys.stdout): - f.close() - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - -class SchemesCMakefile(object): - - header=''' -# All CCPP schemes are defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -set(SCHEMES -''' - footer=''') -''' - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, schemes): - if (self.filename is not sys.stdout): - filepath = os.path.split(self.filename)[0] - if filepath and not os.path.isdir(filepath): - os.makedirs(filepath) - f = open(self.filename, 'w') - else: - f = sys.stdout - - contents = self.header - for scheme in schemes: - contents += ' {0}\n'.format(scheme) - contents += self.footer - f.write(contents) - - if (f is not sys.stdout): - f.close() - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - -class SchemesSourcefile(object): - - header=''' -# All CCPP schemes are defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -export CCPP_SCHEMES="''' - footer='''" -''' - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, schemes): - if (self.filename is not sys.stdout): - filepath = os.path.split(self.filename)[0] - if filepath and not os.path.isdir(filepath): - os.makedirs(filepath) - f = open(self.filename, 'w') - else: - f = sys.stdout - - contents = self.header - for scheme in schemes: - contents += '{0};'.format(scheme) - contents = contents.rstrip(';') - contents += self.footer - f.write(contents) - - if (f is not sys.stdout): - f.close() - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - -class TypedefsMakefile(object): - - header=''' -# All CCPP types are defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -TYPEDEFS =''' - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, typedefs): - if (self.filename is not sys.stdout): - f = open(self.filename, 'w') - else: - f = sys.stdout - - contents = self.header - for typedef in typedefs: - contents += ' \\\n\t {0}'.format(typedef) - f.write(contents) - - if (f is not sys.stdout): - f.close() - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - -class TypedefsCMakefile(object): - - header=''' -# All CCPP types are defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -set(TYPEDEFS -''' - footer=''') -''' - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, typedefs): - if (self.filename is not sys.stdout): - f = open(self.filename, 'w') - else: - f = sys.stdout - - contents = self.header - for typedef in typedefs: - contents += ' {0}\n'.format(typedef) - contents += self.footer - f.write(contents) - - if (f is not sys.stdout): - f.close() - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - -class TypedefsSourcefile(object): - - header=''' -# All CCPP types are defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -export CCPP_TYPEDEFS="''' - footer='''" -''' - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, typedefs): - if (self.filename is not sys.stdout): - filepath = os.path.split(self.filename)[0] - if filepath and not os.path.isdir(filepath): - os.makedirs(filepath) - f = open(self.filename, 'w') - else: - f = sys.stdout - - contents = self.header - for typedef in typedefs: - contents += '{0};'.format(typedef) - contents = contents.rstrip(';') - contents += self.footer - f.write(contents) - - if (f is not sys.stdout): - f.close() - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - -############################################################################### -if __name__ == "__main__": - main() diff --git a/scripts/mkdoc.py b/scripts/mkdoc.py deleted file mode 100755 index 018f88f6..00000000 --- a/scripts/mkdoc.py +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env python3 - -# -# Functions to generate basic documentation in HTML and LaTeX for CCPP metadata -# - -# DH* TODO: create a Python module metadata.py with a class Metadata -# and use this for ccpp_prebuild.py; create to_html and to_latex routines for it - -import logging -import os - -from common import decode_container, escape_tex - -############################################################################### - -def metadata_to_html(metadata, model, filename): - """Create an HTML page with a table that lists each variable provided - by the model. Contrary to metadata_to_latex below, this table does not - include information on variables requested by schemes. The primary use - of the HTML table is to help physics scheme developers to identify the - variables they need when writing a CCPP-compliant scheme.""" - - shading = { 0 : 'darkgray', 1 : 'lightgray' } - success = True - - # Header - html = '''<html> -<title>CCPP variables provided by model {model} - -

CCPP variables provided by model {model}

- - - - - - - - - - - -'''.format(model=model, bgcolor = shading[0]) - - count = 0 - for var_name in sorted(metadata.keys()): - for var in metadata[var_name]: - # Alternate shading, count is 0 1 0 1 ... - count = (count+1) % 2 - # ... create html row ... - line = ''' - - - - - - - - -'''.format(v=var, rank=var.rank.count(':'), container = decode_container(var.container), bgcolor=shading[count]) - html += line - - # Footer - html += '''
standard_namelong_name units rank type kind source {model} name
{v.standard_name}{v.long_name} {v.units} {rank} {v.type} {v.kind} {container} {v.local_name}
- - -''' - - filepath = os.path.split(os.path.abspath(filename))[0] - if not os.path.isdir(filepath): - os.makedirs(filepath) - with open(filename, 'w') as f: - f.write(html) - - logging.info('Metadata table for model {0} written to {1}'.format(model, filename)) - return success - - -def metadata_to_latex(metadata_define, metadata_request, model, filename): - """Create a LaTeX document with a table that lists each variable provided - and/or requested. Uses the GMTB LaTeX templates and style definitons in gmtb.sty.""" - - shading = { 0 : 'darkgray', 1 : 'lightgray' } - success = True - - var_names = sorted(list(set(list(metadata_define.keys()) + list(metadata_request.keys())))) - - styledir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), - '../doc/DevelopersGuide')) - - latex = '''\\documentclass[12pt,letterpaper,oneside,landscape]{{scrbook}} - -\\usepackage{{import}} -\\import{{{styledir}/}}{{gmtb.sty}} -\\renewcommand{{\\thesection}}{{\\arabic{{section}}}} -\\renewcommand{{\\thesubsection}}{{\\arabic{{section}}.\\arabic{{subsection}}}} - -\\begin{{document}} - -\\section{{CCPP variables provided by model {model} vs requested by pool of physics}}\\label{{sec_ccpp_variables}} -\\subsection{{List of variables}} -\\begin{{longtable}}{{l}}'''.format(model=model, styledir=styledir) - - for var_name in var_names: - if var_name in metadata_define.keys(): - var = metadata_define[var_name][0] - else: - var = metadata_request[var_name][0] - line = ''' -\hyperlink{{{standard_name_ref}}}{{\\blue\\underline{{\\execout{{{standard_name}}}}}}} \\\\'''.format( - standard_name=escape_tex(var.standard_name), standard_name_ref=var.standard_name) - latex += line - - latex += ''' -\\end{longtable}\\pagebreak -\\subsection{Description of variables} -{{\\small\\begin{description} -''' - - for var_name in var_names: - if var_name in metadata_define.keys(): - var = metadata_define[var_name][0] - target = escape_tex(decode_container(var.container)) - local_name = escape_tex(var.local_name) - else: - var = metadata_request[var_name][0] - target = 'MISSING' - local_name = 'MISSING' - if var_name in metadata_request.keys(): - requested_list = [ escape_tex(decode_container(v.container)) if v.container else 'none' for v in metadata_request[var_name] ] - # for the purpose of the table, just output the name of the subroutine - for i in range(len(requested_list)): - entry = requested_list[i] - requested_list[i] = entry[entry.find('SUBROUTINE')+len('SUBROUTINE')+1:] - requested = '\\newline '.join(sorted(requested_list)) - else: - requested = 'NOT REQUESTED' - - # Create output - text = ''' -\\begin{{samepage}}\\item{{ -\hypertarget{{{standard_name_ref}}}{{\\blue\\exec{{{standard_name}}}}}}}\\\\ \\nopagebreak -\\begin{{tabular}}{{ll}} -\\execout{{long\_name }} & \\execout{{{long_name} }} \\\\ -\\execout{{units }} & \\execout{{{units} }} \\\\ -\\execout{{rank }} & \\execout{{{rank} }} \\\\ -\\execout{{type }} & \\execout{{{type} }} \\\\ -\\execout{{kind }} & \\execout{{{kind} }} \\\\ -\\execout{{source }} & \\execout{{{target} }} \\\\ -\\execout{{local\_name}} & \\execout{{{local_name} }} \\\\ -\\execout{{requested }} & \\execout{{\\vtop{{{requested}}}}} \\\\ -\\end{{tabular}} -\\vspace{{4pt}} -\\end{{samepage}}'''.format(standard_name=escape_tex(var.standard_name), standard_name_ref=var.standard_name, - long_name=escape_tex(var.long_name), - units=escape_tex(var.units), - rank=var.rank.count(':'), - type=escape_tex(var.type), - kind=escape_tex(var.kind), - target=target, - local_name=local_name, - requested=requested) - latex += text - # Footer - latex += ''' -\\end{description}}} -\\end{document} -''' - - filepath = os.path.split(os.path.abspath(filename))[0] - if not os.path.isdir(filepath): - os.makedirs(filepath) - with open(filename, 'w') as f: - f.write(latex) - - logging.info('Metadata table for model {0} written to {1}'.format(model, filename)) - return success diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py deleted file mode 100755 index e33164a3..00000000 --- a/scripts/mkstatic.py +++ /dev/null @@ -1,2145 +0,0 @@ -#!/usr/bin/env python3 -# - -import collections -import copy -import getopt -import filecmp -import logging -import os -import re -import sys -import types -import xml.etree.ElementTree as ET - -from common import encode_container -from common import lowercase_keys, lowercase_xml -from common import CCPP_STAGES -from common import CCPP_T_INSTANCE_VARIABLE, CCPP_ERROR_CODE_VARIABLE, CCPP_ERROR_MSG_VARIABLE, CCPP_LOOP_COUNTER, CCPP_LOOP_EXTENT -from common import CCPP_BLOCK_NUMBER, CCPP_BLOCK_COUNT, CCPP_BLOCK_SIZES, CCPP_THREAD_NUMBER, CCPP_THREAD_COUNT, CCPP_INTERNAL_VARIABLES -from common import CCPP_HORIZONTAL_LOOP_BEGIN, CCPP_HORIZONTAL_LOOP_END, CCPP_CHUNK_EXTENT -from common import CCPP_CONSTANT_ONE, CCPP_HORIZONTAL_DIMENSION, CCPP_HORIZONTAL_LOOP_EXTENT, CCPP_NUM_INSTANCES -from common import FORTRAN_CONDITIONAL_REGEX_WORDS, FORTRAN_CONDITIONAL_REGEX -from common import CCPP_TYPE, STANDARD_VARIABLE_TYPES, STANDARD_CHARACTER_TYPE -from common import CCPP_STATIC_API_MODULE, CCPP_STATIC_SUBROUTINE_NAME -from metadata_parser import CCPP_MANDATORY_VARIABLES -from mkcap import Var - -############################################################################### - -# Limit suite names to 37 characters; this keeps cap names below 64 characters -# Cap names of 64 characters or longer can cause issues with some compilers. -SUITE_NAME_MAX_CHARS = 37 - -# Maximum number of dimensions of an array allowed by the Fortran 2008 standard -FORTRAN_ARRAY_MAX_DIMS = 15 - -# These variables always need to be present for creating suite and group caps -CCPP_SUITE_VARIABLES = { **CCPP_MANDATORY_VARIABLES, - CCPP_LOOP_COUNTER : Var(local_name = 'loop_cnt', - standard_name = CCPP_LOOP_COUNTER, - long_name = 'loop counter for subcycling loops in CCPP', - units = 'index', - type = 'integer', - dimensions = [], - rank = '', - kind = '', - intent = 'in', - active = 'T', - ), - CCPP_LOOP_EXTENT : Var(local_name = 'loop_max', - standard_name = CCPP_LOOP_EXTENT, - long_name = 'loop counter for subcycling loops in CCPP', - units = 'count', - type = 'integer', - dimensions = [], - rank = '', - kind = '', - intent = 'in', - active = 'T', - ), - } - -# Type and variable declarations for arrays of pointers, required for optional/inactive variables -TMPPTR_ARR_TYPE_DECLARATION = '''type :: {pointer_type_name} - {tmpptr_def} - end type {pointer_type_name}''' -TMPPTR_ARR_DECLARATION = '''type({pointer_type_name}), dimension({dims}) :: {localname}_array''' - -############################################################################### - -def extract_parents_and_indices_from_local_name(local_name): - """Break apart local_name into the different components (members of DDTs) - to determine all variables that are required; this must work for complex - constructs such as Atm(mytile)%q(:,:,:,Atm2(mytile2)%graupel), with - result parent = 'Atm', indices = [mytile, Atm2, mytile2]""" - # First, extract all variables/indices in parentheses (used for subsetting) - indices = [] - while '(' in local_name: - for i in range(len(local_name)): - if local_name[i] == '(': - last_open = i - elif local_name[i] == ')': - last_closed = i - break - index_set = local_name[last_open+1:last_closed].split(',') - for index_group in index_set: - for index in index_group.split(':'): - if index: - if '%' in index: - indices.append(index[:index.find('%')]) - else: - # Skip hard-coded integers that are not variables - try: - int(index) - except ValueError: - indices.append(index) - # Remove this innermost index group (...) from local_name - local_name = local_name.replace(local_name[last_open:last_closed+1], '') - # Remove duplicates from indices - indices = list(set(indices)) - # Derive parent of actual variable (now that all subsets have been processed) - if '%' in local_name: - parent = local_name[:local_name.find('%')] - else: - parent = local_name - # Remove whitespaces - parent = parent.strip() - indices = [ x.strip() for x in indices ] - return (parent, indices) - -def extract_dimensions_from_local_name(local_name): - """Extract the dimensions from a local_name. - Throw away any parent information.""" - # First, find delimiter '%' between parent(s) and child '%' - parent_delimiter_index = -1 - if '%' in local_name: - i = len(local_name)-1 - opened = 0 - while i >= 0: - if local_name[i] == ')': - opened += 1 - elif local_name[i] == '(': - opened -= 1 - elif local_name[i] == '%' and opened == 0: - parent_delimiter_index = i - break - i -= 1 - if '(' in local_name[parent_delimiter_index+1:]: - dim_string_start = local_name[parent_delimiter_index+1:].find('(') - dim_string_end = local_name[parent_delimiter_index+1:].rfind(')') - dim_string = local_name[parent_delimiter_index+1:][dim_string_start:dim_string_end+1] - # Now that we have a dim_string, find all dimensions in this string; - # ignore outermost opening and closing parentheses. - opened = 0 - dim = '' - i = 1 - dimensions = [] - while i <= len(dim_string)-1: - if dim_string[i] == ',' and opened == 0: - dimensions.append(dim) - dim = '' - elif i == len(dim_string)-1 and dim: - dimensions.append(dim) - else: - dim += dim_string[i] - i += 1 - else: - dimensions = [] - dim_string = '' - return (dimensions, dim_string) - -def create_argument_list_wrapped(arguments): - """Create a wrapped argument list, remove trailing ',' """ - argument_list = '' - length = 0 - for argument in arguments: - argument_list += argument + ',' - length += len(argument)+1 - # Split args so that lines don't exceed 260 characters (for PGI) - if length > 70 and not argument == arguments[-1]: - argument_list += ' &\n ' - length = 0 - if argument_list: - argument_list = argument_list.rstrip(',') - return argument_list - -def create_argument_list_wrapped_explicit(arguments, additional_vars_following = False): - """Create a wrapped argument list with explicit arguments x=y. If no additional - variables are added (additional_vars_following == False), remove trailing ',' """ - argument_list = '' - length = 0 - for argument in arguments: - argument_list += argument + '=' + argument + ',' - length += 2*len(argument)+2 - # Split args so that lines don't exceed 260 characters (for PGI) - if length > 70 and not argument == arguments[-1]: - argument_list += ' &\n ' - length = 0 - if argument_list and not additional_vars_following: - argument_list = argument_list.rstrip(',') - return argument_list - -def create_arguments_module_use_var_defs(variable_dictionary, metadata_define, tmpvars = None, tmpptrs = None): - """Given a dictionary of standard names and variables, and a metadata - dictionary with the variable definitions by the host model, create a list - of arguments (local names), module use statements (for derived data types - and non-standard kinds), and the variable definition statements.""" - arguments = [] - module_use = [] - var_defs = [] - local_kind_and_type_vars = [] - local_pointer_type_defs = [] - - # We need to run through this loop twice. In the first pass, process all scalars. - # In the second pass, process all arrays. This is so that any potential dimension - # that is used in the following array variable definitions is defined first to avoid - # violating the Fortran 2008 standard. - # https://community.intel.com/t5/Intel-Fortran-Compiler/Order-of-declaration-statements-with-and-without-implicit-typing/td-p/1176155 - iteration = 1 - while iteration <= 2: - for standard_name in variable_dictionary.keys(): - if iteration == 1 and variable_dictionary[standard_name].dimensions: - continue - elif iteration == 2 and not variable_dictionary[standard_name].dimensions: - continue - # Add variable local name and variable definitions - arguments.append(variable_dictionary[standard_name].local_name) - var_defs.append(variable_dictionary[standard_name].print_def_intent(metadata_define)) - # Add special kind variables and derived data type definitions to module use statements - if variable_dictionary[standard_name].type in STANDARD_VARIABLE_TYPES and variable_dictionary[standard_name].kind \ - and not variable_dictionary[standard_name].type == STANDARD_CHARACTER_TYPE: - kind_var_standard_name = variable_dictionary[standard_name].kind - if not kind_var_standard_name in local_kind_and_type_vars: - if not kind_var_standard_name in metadata_define.keys(): - raise Exception("Kind {kind}, required by {std_name}, not defined by host model".format( - kind=kind_var_standard_name, std_name=standard_name)) - kind_var = metadata_define[kind_var_standard_name][0] - module_use.append(kind_var.print_module_use()) - local_kind_and_type_vars.append(kind_var_standard_name) - elif not variable_dictionary[standard_name].type in STANDARD_VARIABLE_TYPES: - type_var_standard_name = variable_dictionary[standard_name].type - if not type_var_standard_name in local_kind_and_type_vars: - if not type_var_standard_name in metadata_define.keys(): - raise Exception("Type {type}, required by {std_name}, not defined by host model".format( - type=type_var_standard_name, std_name=standard_name)) - type_var = metadata_define[type_var_standard_name][0] - module_use.append(type_var.print_module_use()) - local_kind_and_type_vars.append(type_var_standard_name) - iteration += 1 - - # Add any local variables (required for unit conversions, array transformations, ...), - # and add any local pointers (required for conditionally allocated arrays) - if tmpvars or tmpptrs: - var_defs.append('') - var_defs.append('! Local variables/pointers for unit conversions, array transformations, ...') - for tmpvar in list(tmpvars) + list(tmpptrs): - # Regular variables - if tmpvar in list(tmpvars): - var_defs.append(tmpvar.print_def_local(metadata_define)) - # Pointers are more complicated - else: - if tmpvar.type == 'character' and 'len=' in tmpvar.kind: - pointer_type_name = f"{tmpvar.type}_{tmpvar.kind.replace('=','')}_r{len(tmpvar.dimensions)}_ptr_arr_type" - elif tmpvar.kind: - pointer_type_name = f"{tmpvar.type}_{tmpvar.kind}_rank{len(tmpvar.dimensions)}_ptr_arr_type" - else: - pointer_type_name = f"{tmpvar.type}_default_kind_rank{len(tmpvar.dimensions)}_ptr_arr_type" - if not pointer_type_name in local_pointer_type_defs: - var_defs.append(TMPPTR_ARR_TYPE_DECLARATION.format(pointer_type_name=pointer_type_name, - tmpptr_def=tmpvar.print_def_local(metadata_define))) - local_pointer_type_defs.append(pointer_type_name) - var_defs.append(TMPPTR_ARR_DECLARATION.format(pointer_type_name=pointer_type_name, - dims=f'1:{CCPP_INTERNAL_VARIABLES[CCPP_THREAD_COUNT]}', localname=tmpvar.local_name)) - # Add special kind variables - if tmpvar.type in STANDARD_VARIABLE_TYPES and tmpvar.kind and not tmpvar.type == STANDARD_CHARACTER_TYPE: - kind_var_standard_name = tmpvar.kind - if not kind_var_standard_name in local_kind_and_type_vars: - if not kind_var_standard_name in metadata_define.keys(): - raise Exception("Kind {kind} not defined by host model".format(kind=kind_var_standard_name)) - kind_var = metadata_define[kind_var_standard_name][0] - module_use.append(kind_var.print_module_use()) - local_kind_and_type_vars.append(kind_var_standard_name) - - return (arguments, module_use, var_defs) - -class API(object): - - header=''' -! -! This work (Common Community Physics Package), identified by NOAA, NCAR, -! CU/CIRES, is free of known copyright restrictions and is placed in the -! public domain. -! -! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -! THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -! - -!> -!! @brief Auto-generated API for the CCPP static build -!! -! -module {module} - -{module_use} - implicit none - - private - public :: {subroutines} - - contains - - ! Necessary to convert incoming suite and group names to lowercase - function to_lower(str) result(lower) - implicit none - character(len=*), intent(in) :: str - character(len=len(str)) :: lower - integer, parameter :: upper_to_lower = ichar('a') - ichar('A') - integer :: i, ichar_val - - do i = 1, len(str) - ichar_val = ichar(str(i:i)) - if (ichar_val >= ichar('A') .and. ichar_val <= ichar('Z')) then - lower(i:i) = char(ichar_val + upper_to_lower) - else - lower(i:i) = str(i:i) - end if - end do - end function to_lower -''' - - sub = ''' - subroutine {subroutine}({ccpp_var_name}, suite_name, group_name, ierr) - - use ccpp_types, only : ccpp_t - - implicit none - - type(ccpp_t), intent(inout) :: {ccpp_var_name} - character(len=*), intent(in) :: suite_name - character(len=*), optional, intent(in) :: group_name - integer, intent(out) :: ierr - - ierr = 0 - -{suite_switch} - else - - write({ccpp_var_name}%errmsg,'(*(a))') 'Invalid suite ' // to_lower(trim(suite_name)) - ierr = 1 - - end if - - {ccpp_var_name}%errflg = ierr - - end subroutine {subroutine} -''' - - footer = ''' -end module {module} -''' - - def __init__(self, **kwargs): - self._filename = CCPP_STATIC_API_MODULE + '.F90' - self._module = CCPP_STATIC_API_MODULE - self._subroutines = None - self._suites = [] - self._directory = '.' - self._update_api = True - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - @property - def filename(self): - '''Get the filename to write API to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - - @property - def directory(self): - '''Get the directory to write API to.''' - return self._directory - - @directory.setter - def directory(self, value): - self._directory = value - - @property - def update_api(self): - '''Get the update_api flag.''' - return self._update_api - - @update_api.setter - def update_api(self, value): - self._update_api = value - - @property - def module(self): - '''Get the module name of the API.''' - return self._module - - @module.setter - def module(self, value): - self._module = value - - @property - def subroutines(self): - '''Get the subroutines names of the API to.''' - return self._subroutines - - def write(self): - """Write API for static build""" - if not self._suites: - raise Exception("No suites specified for generating API") - suites = self._suites - - # Module use statements for suite and group caps - module_use = '' - for suite in suites: - for subroutine in suite.subroutines: - module_use += ' use {module}, only: {subroutine}\n'.format(module=suite.module, subroutine=subroutine) - for group in suite.groups: - for subroutine in group.subroutines: - module_use += ' use {module}, only: {subroutine}\n'.format(module=group.module, subroutine=subroutine) - - # Add all variables required to module use statements. This is for the API only, - # because the static API imports all variables from modules instead of receiving them - # via the argument list. Special handling for a single variable of type CCPP_TYPE (ccpp_t), - # which comes in as a scalar for any potential block/thread via the argument list. - ccpp_var = None - parent_standard_names = [] - for ccpp_stage in CCPP_STAGES.keys(): - for suite in suites: - for parent_standard_name in suite.parents[ccpp_stage].keys(): - if not parent_standard_name in parent_standard_names: - parent_var = suite.parents[ccpp_stage][parent_standard_name] - # Identify which variable is of type CCPP_TYPE (need local name) - if parent_var.type == CCPP_TYPE: - if ccpp_var and not ccpp_var.local_name==parent_var.local_name: - raise Exception('There can be only one variable of type {0}, found {1} and {2}'.format( - CCPP_TYPE, ccpp_var.local_name, parent_var.local_name)) - ccpp_var = parent_var - continue - module_use += ' {0}\n'.format(parent_var.print_module_use()) - parent_standard_names.append(parent_standard_name) - if not ccpp_var: - raise Exception('No variable of type {0} found - need a scalar instance.'.format(CCPP_TYPE)) - elif not ccpp_var.rank == '': - raise Exception('CCPP variable {0} of type {1} must be a scalar.'.format(ccpp_var.local_name, CCPP_TYPE)) - del parent_standard_names - - # Create a subroutine for each stage - self._subroutines=[] - subs = '' - for ccpp_stage in CCPP_STAGES.keys(): - suite_switch = '' - for suite in suites: - # Calls to groups of schemes for this stage - group_calls = '' - for group in suite.groups: - # The and groups require special treatment, - # since they can only be run in the respective stage (init/finalize) - if (group.init and not ccpp_stage == 'init') or \ - (group.finalize and not ccpp_stage == 'finalize'): - continue - if not group_calls: - clause = 'if' - else: - clause = 'else if' - argument_list_group = create_argument_list_wrapped_explicit(group.arguments[ccpp_stage]) - group_calls += ''' - {clause} (to_lower(trim(group_name))=="{group_name}") then - ierr = {suite_name}_{group_name}_{stage}_cap({arguments})'''.format(clause=clause, - suite_name=group.suite, - group_name=group.name, - stage=CCPP_STAGES[ccpp_stage], - arguments=argument_list_group) - group_calls += ''' - else - write({ccpp_var_name}%errmsg, '(*(a))') 'Group ' // to_lower(trim(group_name)) // ' not found' - ierr = 1 - end if -'''.format(ccpp_var_name=ccpp_var.local_name, group_name=group.name) - - # Call to entire suite for this stage - - # Create argument list for calling the full suite - argument_list_suite = create_argument_list_wrapped_explicit(suite.arguments[ccpp_stage]) - suite_call = ''' - ierr = {suite_name}_{stage}_cap({arguments}) -'''.format(suite_name=suite.name, stage=CCPP_STAGES[ccpp_stage], arguments=argument_list_suite) - - # Add call to all groups of this suite and to the entire suite - if not suite_switch: - clause = 'if' - else: - clause = 'else if' - suite_switch += ''' - {clause} (to_lower(trim(suite_name))=="{suite_name}") then - - if (present(group_name)) then -{group_calls} - else -{suite_call} - end if -'''.format(clause=clause, suite_name=suite.name, group_calls=group_calls, suite_call=suite_call) - - subroutine = CCPP_STATIC_SUBROUTINE_NAME.format(stage=ccpp_stage) - self._subroutines.append(subroutine) - subs += API.sub.format(subroutine=subroutine, - ccpp_var_name=ccpp_var.local_name, - suite_switch=suite_switch) - - # Write output to stdout or file - if (self.filename is not sys.stdout): - filepath = os.path.split(self.filename)[0] - if filepath and not os.path.isdir(filepath): - os.makedirs(filepath) - # If the file exists, write to temporary file first and compare them: - # - if identical, delete the temporary file and keep the existing one - # and set the API update flag to false - # - if different, replace existing file with temporary file and set - # the API update flag to true (default value) - # - always replace the file if any of the suite caps has changed - # If the file does not exist, write the API an set the flag to true - if os.path.isfile(self.filename) and \ - not any([suite.update_cap for suite in suites]): - write_to_test_file = True - test_filename = self.filename + '.test' - f = open(test_filename, 'w') - else: - write_to_test_file = False - f = open(self.filename, 'w') - else: - f = sys.stdout - f.write(API.header.format(module=self._module, - module_use=module_use, - subroutines=','.join(self._subroutines))) - f.write(subs) - f.write(Suite.footer.format(module=self._module)) - if (f is not sys.stdout): - f.close() - # See comment above on updating the API or not - if write_to_test_file: - if filecmp.cmp(self.filename, test_filename): - # Files are equal, delete the test API and set update flag to False - os.remove(test_filename) - self.update_api = False - else: - # Files are different, replace existing API with - # the test API and set update flag to True - # Python 3 only: os.replace(test_filename, self.filename) - os.remove(self.filename) - os.rename(test_filename, self.filename) - self.update_api = True - else: - self.update_api = True - return - - def write_includefile(self, source_filename, type): - success = True - filepath = os.path.split(source_filename)[0] - if filepath and not os.path.isdir(filepath): - os.makedirs(filepath) - # If the file exists, write to temporary file first and compare them: - # - if identical, delete the temporary file and keep the existing one - # - if different, replace existing file with temporary file - # - however, always replace the file if the API update flag is true - if os.path.isfile(source_filename) and not self.update_api: - write_to_test_file = True - test_filename = source_filename + '.test' - f = open(test_filename, 'w') - else: - write_to_test_file = False - f = open(source_filename, 'w') - - if type == 'shell': - # Contents of shell/source file - contents = """# The CCPP static API is defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -export CCPP_STATIC_API=\"{filename}\" -""".format(filename=os.path.abspath(os.path.join(self.directory,self.filename))) - elif type == 'cmake': - # Contents of cmake include file - contents = """# The CCPP static API is defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -set(API \"{filename}\") -""".format(filename=os.path.abspath(os.path.join(self.directory,self.filename))) - else: - logging.error('Encountered unknown type of file "{type}" when writing include file for static API'.format(type=type)) - success = False - return - - f.write(contents) - f.close() - # See comment above on updating the API or not - if write_to_test_file: - if filecmp.cmp(source_filename, test_filename): - # Files are equal, delete the test file - os.remove(test_filename) - else: - # Files are different, replace existing file - # Python 3 only: os.replace(test_filename, source_filename) - os.remove(source_filename) - os.rename(test_filename, source_filename) - return success - - -class Suite(object): - - header=''' -! -! This work (Common Community Physics Package), identified by NOAA, NCAR, -! CU/CIRES, is free of known copyright restrictions and is placed in the -! public domain. -! -! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -! THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -! - -!> -!! @brief Auto-generated cap module for the CCPP suite -!! -! -module {module} - -{module_use} - - implicit none - - private - public :: {subroutines} - - contains -''' - - sub = ''' - function {subroutine}({arguments}) result(ierr) - - {module_use} - - implicit none - - integer :: ierr - {var_defs} - - ierr = 0 - -{body} - - end function {subroutine} -''' - - footer = ''' -end module {module} -''' - - def __init__(self, **kwargs): - self._name = None - self._filename = sys.stdout - self._sdf_name = None - self._all_schemes_called = None - self._all_subroutines_called = None - self._call_tree = {} - self._caps = None - self._module = None - self._subroutines = None - self._parents = { ccpp_stage : collections.OrderedDict() for ccpp_stage in CCPP_STAGES.keys() } - self._arguments = { ccpp_stage : [] for ccpp_stage in CCPP_STAGES.keys() } - self._update_cap = True - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - @property - def name(self): - '''Get the name of the suite.''' - return self._name - - @property - def sdf_name(self): - '''Get the name of the suite definition file.''' - return self._sdf_name - - @sdf_name.setter - def sdf_name(self, value): - self._sdf_name = value - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - - @property - def update_cap(self): - '''Get the update_cap flag.''' - return self._update_cap - - @update_cap.setter - def update_cap(self, value): - self._update_cap = value - - def parse(self, make_call_tree=False): - '''Parse the suite definition file.''' - success = True - - if not os.path.exists(self._sdf_name): - logging.critical("Suite definition file {0} not found.".format(self._sdf_name)) - success = False - return success - - tree = ET.parse(self._sdf_name) - suite_xml = lowercase_xml(tree.getroot()) - self._name = suite_xml.get('name') - - # Check if suite name is too long - if len(self._name) > SUITE_NAME_MAX_CHARS: - logging.critical(f"Suite name {self._name} has more than the allowed {SUITE_NAME_MAX_CHARS} characters") - success = False - return success - - # Flattened lists of all schemes and subroutines in SDF - self._all_schemes_called = [] - self._all_subroutines_called = [] - - if make_call_tree: - # Call tree of all schemes in SDF. call_tree is a dictionary, with keys corresponding to each group in a suite, and - # the value associated with each key being an ordered list of the schemes in each group (with duplicates and subcycles) - self._call_tree = {} - - # Build hierarchical structure as in SDF - self._groups = [] - for group_xml in suite_xml: - subcycles = [] - - self._call_tree[group_xml.attrib['name']] = [] - # Add suite-wide init scheme to group 'init', similar for finalize - if group_xml.tag == 'init' or group_xml.tag == 'finalize': - self._all_schemes_called.append(group_xml.text) - self._all_subroutines_called.append(group_xml.text + '_' + group_xml.tag) - schemes = [group_xml.text] - subcycles.append(Subcycle(loop=1, schemes=schemes)) - if group_xml.tag == 'init': - self._groups.append(Group(name=group_xml.tag, subcycles=subcycles, suite=self._name, init=True)) - elif group_xml.tag == 'finalize': - self._groups.append(Group(name=group_xml.tag, subcycles=subcycles, suite=self._name, finalize=True)) - continue - - # Parse subcycles of all regular groups - for subcycle_xml in group_xml: - schemes = [] - for scheme_xml in subcycle_xml: - self._all_schemes_called.append(scheme_xml.text) - schemes.append(scheme_xml.text) - loop=int(subcycle_xml.get('loop')) - for ccpp_stage in CCPP_STAGES: - self._all_subroutines_called.append(scheme_xml.text + '_' + CCPP_STAGES[ccpp_stage]) - - subcycles.append(Subcycle(loop=loop, schemes=schemes)) - - if make_call_tree: - # Populate call tree from SDF's heirarchical structure, including multiple calls in subcycle loops - for loop in range(0,int(subcycle_xml.get('loop'))): - for scheme_xml in subcycle_xml: - self._call_tree[group_xml.attrib['name']].append(scheme_xml.text) - - self._groups.append(Group(name=group_xml.get('name'), subcycles=subcycles, suite=self._name)) - - # Remove duplicates from list of all subroutines an schemes - self._all_schemes_called = list(set(self._all_schemes_called)) - self._all_subroutines_called = list(set(self._all_subroutines_called)) - - return success - - def print_debug(self): - '''Basic debugging output about the suite.''' - print("ALL SUBROUTINES:") - print(self._all_subroutines_called) - print("STRUCTURED:") - print(self._groups) - for group in self._groups: - group.print_debug() - - @property - def all_schemes_called(self): - '''Get the list of all schemes.''' - return self._all_schemes_called - - @property - def call_tree(self): - '''Get the call tree of the suite (all schemes, in order, with duplicates and loops).''' - return self._call_tree - - @property - def all_subroutines_called(self): - '''Get the list of all subroutines.''' - return self._all_subroutines_called - - @property - def module(self): - '''Get the list of the module generated for this suite.''' - return self._module - - @property - def subroutines(self): - '''Get the list of all subroutines generated for this suite.''' - return self._subroutines - - @property - def caps(self): - '''Get the list of all caps.''' - return self._caps - - @property - def groups(self): - '''Get the list of groups in this suite.''' - return self._groups - - @property - def parents(self): - '''Get the parent variables for the suite.''' - return self._parents - - @parents.setter - def parents(self, value): - self._parents = value - - @property - def arguments(self): - '''Get the argument list for the suite.''' - return self._arguments - - @arguments.setter - def arguments(self, value): - self._arguments = value - - def write(self, metadata_request, metadata_define, arguments, debug): - """Create caps for all groups in the suite and for the entire suite - (calling the group caps one after another). Add additional code for - debugging if debug flag is True.""" - # Set name of module and filename of cap - self._module = 'ccpp_{suite_name}_cap'.format(suite_name=self._name) - self.filename = '{module_name}.F90'.format(module_name=self._module) - # Init - self._subroutines = [] - # Write group caps and generate module use statements; combine the argument lists - # and variable definitions for all groups into a suite argument list. This may - # require adjusting the intent of the variables. - module_use = '' - for group in self._groups: - group.write(metadata_request, metadata_define, arguments, debug) - for subroutine in group.subroutines: - module_use += ' use {m}, only: {s}\n'.format(m=group.module, s=subroutine) - for ccpp_stage in CCPP_STAGES.keys(): - for parent_standard_name in group.parents[ccpp_stage].keys(): - if parent_standard_name in self.parents[ccpp_stage]: - if self.parents[ccpp_stage][parent_standard_name].intent == 'in' and \ - not group.parents[ccpp_stage][parent_standard_name].intent == 'in': - self.parents[ccpp_stage][parent_standard_name].intent = 'inout' - elif self.parents[ccpp_stage][parent_standard_name].intent == 'out' and \ - not group.parents[ccpp_stage][parent_standard_name].intent == 'out': - self.parents[ccpp_stage][parent_standard_name].intent = 'inout' - else: - self.parents[ccpp_stage][parent_standard_name] = copy.deepcopy(group.parents[ccpp_stage][parent_standard_name]) - subs = '' - for ccpp_stage in CCPP_STAGES.keys(): - # Create a wrapped argument list for calling the suite, - # get module use statements and variable definitions - (self.arguments[ccpp_stage], sub_module_use, sub_var_defs) = \ - create_arguments_module_use_var_defs(self.parents[ccpp_stage], metadata_define) - argument_list_suite = create_argument_list_wrapped(self.arguments[ccpp_stage]) - body = '' - for group in self._groups: - # Groups 'init'/'finalize' are only run in stages 'init'/'finalize' - if (group.init and not ccpp_stage == 'init') or \ - (group.finalize and not ccpp_stage == 'finalize'): - continue - # Create a wrapped argument list for calling the group - (arguments_group, dummy, dummy) = create_arguments_module_use_var_defs(group.parents[ccpp_stage], metadata_define) - argument_list_group = create_argument_list_wrapped_explicit(arguments_group) - - # Write to body that calls the groups for this stage - body += ''' - ierr = {suite_name}_{group_name}_{stage}_cap({arguments}) - if (ierr/=0) return -'''.format(suite_name=self._name, group_name=group.name, stage=CCPP_STAGES[ccpp_stage], arguments=argument_list_group) - # Add name of subroutine in the suite cap to list of subroutine names - subroutine = '{name}_{stage}_cap'.format(name=self._name, stage=CCPP_STAGES[ccpp_stage]) - self._subroutines.append(subroutine) - # Add subroutine to output - subs += Suite.sub.format(subroutine=subroutine, - arguments=argument_list_suite, - module_use='\n '.join(sub_module_use), - var_defs='\n '.join(sub_var_defs), - body=body) - - # Write cap to stdout or file - if (self.filename is not sys.stdout): - filepath = os.path.split(self.filename)[0] - if filepath and not os.path.isdir(filepath): - os.makedirs(filepath) - # If the file exists, write to temporary file first and compare them: - # - if identical, delete the temporary file and keep the existing one - # and set the suite cap update flag to false - # - if different, replace existing file with temporary file and set - # the suite cap update flag to true (default value) - # - however, if any of the group caps has changed, rewrite the suite - # cap as well and set the suite cap update flag to true - # If the file does not exist, write the cap an set the flag to true - if os.path.isfile(self.filename) and \ - not any([group.update_cap for group in self._groups]): - write_to_test_file = True - test_filename = self.filename + '.test' - f = open(test_filename, 'w') - else: - write_to_test_file = False - f = open(self.filename, 'w') - else: - f = sys.stdout - f.write(Suite.header.format(module=self._module, - module_use=module_use, - subroutines=', &\n '.join(self._subroutines))) - f.write(subs) - f.write(Suite.footer.format(module=self._module)) - if (f is not sys.stdout): - f.close() - # See comment above on updating the suite cap or not - if write_to_test_file: - if filecmp.cmp(self.filename, test_filename): - # Files are equal, delete the test cap - # and set update flag to False - os.remove(test_filename) - self.update_cap = False - else: - # Files are different, replace existing cap - # with test cap and set flag to True - # Python 3 only: os.replace(test_filename, self.filename) - os.remove(self.filename) - os.rename(test_filename, self.filename) - self.update_cap = True - else: - self.update_cap = True - - # Create list of all caps generated (for groups and suite) - self._caps = [ self.filename ] - for group in self._groups: - self._caps.append(group.filename) - - -############################################################################### -class Group(object): - - header=''' -! -! This work (Common Community Physics Package), identified by NOAA, NCAR, -! CU/CIRES, is free of known copyright restrictions and is placed in the -! public domain. -! -! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -! THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -! - -!> -!! @brief Auto-generated cap module for the CCPP {group} group -!! -! -module {module} - -{module_use} - - implicit none - - private - public :: {subroutines} - - logical, dimension({num_instances}), save :: initialized = .false. - - contains -''' - - sub = ''' - function {subroutine}({argument_list}) result(ierr) - - {module_use} - - implicit none - - ! Error handling - integer :: ierr - - {var_defs} - - ierr = 0 - -{initialized_test_block} - -{body} - -{initialized_set_block} - - end function {subroutine} -''' - - footer = ''' -end module {module} -''' - - initialized_test_blocks = { - 'init' : ''' - if (initialized({ccpp_var_name}%ccpp_instance)) return -''', - 'timestep_init' : ''' - if (.not.initialized({ccpp_var_name}%ccpp_instance)) then - write({target_name_msg},'(*(a))') '{name}_timestep_init called before {name}_init' - {target_name_flag} = 1 - return - end if -''', - 'run' : ''' - if (.not.initialized({ccpp_var_name}%ccpp_instance)) then - write({target_name_msg},'(*(a))') '{name}_run called before {name}_init' - {target_name_flag} = 1 - return - end if -''', - 'timestep_finalize' : ''' - if (.not.initialized({ccpp_var_name}%ccpp_instance)) then - write({target_name_msg},'(*(a))') '{name}_timestep_finalize called before {name}_init' - {target_name_flag} = 1 - return - end if -''', - 'finalize' : ''' - if (.not.initialized({ccpp_var_name}%ccpp_instance)) return -''', - } - - initialized_set_blocks = { - 'init' : ''' - initialized({ccpp_var_name}%ccpp_instance) = .true. -''', - 'timestep_init' : '', - 'run' : '', - 'timestep_finalize' : '', - 'finalize' : ''' - initialized = .false. -''', - } - - def __init__(self, **kwargs): - self._name = '' - self._suite = None - self._filename = sys.stdout - self._init = False - self._finalize = False - self._module = None - self._subroutines = None - self._pset = None - self._parents = { ccpp_stage : collections.OrderedDict() for ccpp_stage in CCPP_STAGES } - self._arguments = { ccpp_stage : [] for ccpp_stage in CCPP_STAGES } - self._update_cap = True - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, metadata_request, metadata_define, arguments_in, debug): - """Create caps for all stages of this group. Add additional code for - debugging if debug flag is True.""" - - # First, convert all keys in arguments_in to lowercase (recursively) - arguments = lowercase_keys(arguments_in) - - # Create an inverse lookup table of local variable names defined (by the host model) and standard names - standard_name_by_local_name_define = collections.OrderedDict() - for standard_name in metadata_define.keys(): - standard_name_by_local_name_define[metadata_define[standard_name][0].local_name] = standard_name - - # First get target names of standard CCPP variables for subcycling and error handling - ccpp_loop_counter_target_name = metadata_request[CCPP_LOOP_COUNTER][0].target - ccpp_loop_extent_target_name = metadata_request[CCPP_LOOP_EXTENT][0].target - ccpp_error_code_target_name = metadata_request[CCPP_ERROR_CODE_VARIABLE][0].target - ccpp_error_msg_target_name = metadata_request[CCPP_ERROR_MSG_VARIABLE][0].target - # Then, identify the variable name of the mandatory ccpp_t variable defined by the host model - ccpp_var = metadata_define[CCPP_T_INSTANCE_VARIABLE][0] - - # Init - module_use = '' - self._module = 'ccpp_{suite}_{name}_cap'.format(name=self._name, suite=self._suite) - self._filename = '{module_name}.F90'.format(module_name=self._module) - self._subroutines = [] - local_subs = '' - # - for ccpp_stage in CCPP_STAGES.keys(): - # The special init and finalize routines are only run in that stage - if self._init and not ccpp_stage == 'init': - continue - elif self._finalize and not ccpp_stage == 'finalize': - continue - # For mapping local variable names to standard names - local_vars = collections.OrderedDict() - # For mapping temporary variable names (for unit conversions, etc) to local variable names - tmpvar_cnt = 0 - tmpvars = collections.OrderedDict() - # For mapping temporary pointer names (for potentially unallocated arrays) to local variable names - tmpptr_cnt = 0 - tmpptrs = collections.OrderedDict() - # - body = '' - # Variable definitions automatically added for subroutines - var_defs = '' - # List of manual variable definitions, for example for handling blocked data structures - var_defs_manual = [] - # Conditionals for variables (used or allocated only under certain conditions) - conditionals = {} - # - for subcycle in self._subcycles: - subcycle_body = '' - # Call all schemes - for scheme_name in subcycle.schemes: - # actions_before and actions_after capture operations such - # as unit conversions, transformations that have to happen - # before and/or after the call to the subroutine (scheme) - actions_before = '' - actions_after = '' - # - module_name = scheme_name - subroutine_name = scheme_name + '_' + ccpp_stage - container = encode_container(module_name, scheme_name, subroutine_name) - # Skip entirely empty routines or non-existent routines - if not subroutine_name in arguments[scheme_name].keys() or not arguments[scheme_name][subroutine_name]: - continue - error_check = '' - args = '' - length = 0 - - # First, add a few mandatory variables to the list of required - # variables. This is mostly for handling horizontal dimensions - # correctly for the different CCPP phases and for chunked arrays - additional_variables_required = [] - if CCPP_HORIZONTAL_LOOP_EXTENT in metadata_define.keys(): - for add_var in [ CCPP_CONSTANT_ONE, CCPP_HORIZONTAL_LOOP_EXTENT]: - if not add_var in local_vars.keys() \ - and not add_var in additional_variables_required + arguments[scheme_name][subroutine_name]: - logging.debug("Adding variable {} for handling blocked data structures".format(add_var)) - additional_variables_required.append(add_var) - elif ccpp_stage == 'run' and \ - CCPP_HORIZONTAL_LOOP_BEGIN in metadata_define.keys() and \ - CCPP_HORIZONTAL_LOOP_END in metadata_define.keys() and \ - CCPP_CHUNK_EXTENT in metadata_define.keys(): - for add_var in [ CCPP_HORIZONTAL_LOOP_BEGIN, CCPP_HORIZONTAL_LOOP_END, CCPP_CHUNK_EXTENT]: - if not add_var in local_vars.keys() \ - and not add_var in additional_variables_required + arguments[scheme_name][subroutine_name]: - logging.debug("Adding variable {} for handling chunked data arrays".format(add_var)) - additional_variables_required.append(add_var) - # Next, identify all dimensions needed to handle the arguments - # and add them to the list of required variables for the cap - for var_standard_name in arguments[scheme_name][subroutine_name]: - if not var_standard_name in metadata_define.keys(): - raise Exception('Variable {standard_name} not defined in host model metadata'.format( - standard_name=var_standard_name)) - var = metadata_define[var_standard_name][0] - # dim_expression can be 'A', '1', '1:A', ... - for dim_expression in var.dimensions: - dims = dim_expression.split(':') - for dim in dims: - dim = dim.lower() - try: - dim = int(dim) - except ValueError: - if not dim in local_vars.keys() and \ - not dim in additional_variables_required + arguments[scheme_name][subroutine_name]: - if not dim in metadata_define.keys(): - raise Exception('Dimension {}, required by variable {}, not defined in host model metadata'.format( - dim, var_standard_name)) - logging.debug("Adding dimension {} for variable {}".format(dim, var_standard_name)) - additional_variables_required.append(dim) - - # If blocked data structures need to be converted, add necessary variables - if ccpp_stage in ['init', 'timestep_init', 'timestep_finalize', 'finalize'] and CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER] in var.local_name: - for add_var in [ CCPP_BLOCK_COUNT, CCPP_HORIZONTAL_DIMENSION]: - if not add_var in local_vars.keys() \ - and not add_var in additional_variables_required + arguments[scheme_name][subroutine_name]: - logging.debug("Adding variable {} for handling blocked data structures".format(add_var)) - additional_variables_required.append(add_var) - - # If the variable is only active/used under certain conditions, add necessary variables - # also record the conditional for later use in unit conversions / blocked data conversions. - if var.active == 'T': - conditional = '.true.' - elif var.active == 'F': - conditional = '.false.' - else: - # Convert conditional expression in standard_name format to local names known to the host model - conditional = '' - # Find all words in the conditional, for each of them look for a matching - # standard name in the list of known variables - items = FORTRAN_CONDITIONAL_REGEX.findall(var.active) - for item in items: - item = item.lower() - if item in FORTRAN_CONDITIONAL_REGEX_WORDS: - conditional += item - else: - # Detect integers, following Python's "easier to ask forgiveness than permission" mentality - try: - int(item) - conditional += item - except ValueError: - if not item in metadata_define.keys(): - raise Exception("Variable {} used in conditional for {} not known to host model".format( - item, var_standard_name)) - var2 = metadata_define[item][0] - conditional += var2.local_name - # Add to list of required variables for the cap - if not item in local_vars.keys() \ - and not item in additional_variables_required + arguments[scheme_name][subroutine_name]: - logging.debug("Adding variable {} for handling conditionals".format(item)) - additional_variables_required.append(item) - # Conditionals are identical per requirement, no need to test for consistency again - if not var_standard_name in conditionals.keys(): - conditionals[var_standard_name] = conditional - - # Extract all variables needed (including indices for components/slices of arrays and - # including their parents). We need to run this twice, because the dimensions of parent - # variables get added to additional_variables_required in the first pass. - iteration = 1 - while iteration <= 2: - for var_standard_name in additional_variables_required + arguments[scheme_name][subroutine_name]: - # Pick the correct variable for this module/scheme/subroutine - # from the list of requested variables, if it is in that list - if var_standard_name in arguments[scheme_name][subroutine_name]: - for var in metadata_request[var_standard_name]: - if container == var.container: - break - # This is a dimension or required variable added automatically (e.g. for handling blocked data) - else: - # Create a copy of the variable in the metadata dictionary - # of host model variables and set necessary default values - var = copy.deepcopy(metadata_define[var_standard_name][0]) - var.intent = 'in' - - if not var_standard_name in local_vars.keys(): - # The full name of the variable as known to the host model - var_local_name_define = metadata_define[var_standard_name][0].local_name - - # Break apart var_local_name_define into the different components (members of DDTs) - # to determine all variables that are required - (parent_local_name_define, parent_local_names_define_indices) = \ - extract_parents_and_indices_from_local_name(var_local_name_define) - - parent_standard_name = None - parent_var = None - # Check for each of the derived parent local names as defined by the host model - # if they are registered (i.e. if there is a standard name for it). Note that - # the output of extract_parents_and_indices_from_local_name is stripped of any - # array subset information, i.e. a local name 'Atm(:)%...' will produce a - # parent local name 'Atm'. Since the rank of the parent variable is not known - # at this point and since the local name in the host model metadata table could - # contain '(:)', '(:,:)', ... (up to the rank of the array), we search for the - # maximum number of dimensions allowed by the Fortran standard. - for local_name_define in [parent_local_name_define] + parent_local_names_define_indices: - parent_standard_name = None - parent_var = None - for i in range(FORTRAN_ARRAY_MAX_DIMS+1): - if i==0: - dims_string = '' - else: - # (:) for i==1, (:,:) for i==2, ... - dims_string = '(' + ','.join([':' for j in range(i)]) + ')' - if local_name_define+dims_string in standard_name_by_local_name_define.keys(): - parent_standard_name = standard_name_by_local_name_define[local_name_define+dims_string] - parent_var = metadata_define[parent_standard_name][0] - break - if not parent_var: - raise Exception('Parent variable {parent} of {child} with standard name '.format( - parent=local_name_define, child=var_local_name_define)+\ - '{standard_name} not defined in host model metadata'.format( - standard_name=var_standard_name)) - - # Reset local name for entire array to a notation without (:), (:,:), etc.; - # this is needed for the var.print_def_intent() routine to work correctly - parent_var.local_name = local_name_define - - # Add the parent_var's dimensions to the locally defined dimensions: - # dim_expression can be 'A', '1', '1:A', ... - for dim_expression in parent_var.dimensions: - dims = dim_expression.split(':') - for dim in dims: - dim = dim.lower() - try: - dim = int(dim) - except ValueError: - if not dim in local_vars.keys() and \ - not dim in additional_variables_required + arguments[scheme_name][subroutine_name]: - if not dim in metadata_define.keys(): - raise Exception('Dimension {}, required by parent variable {}, not defined in host model metadata'.format( - dim, parent_standard_name)) - logging.debug("Adding dimension {} for parent variable {}".format(dim, parent_standard_name)) - additional_variables_required.append(dim) - - # Add variable to dictionary of parent variables, if not already there. - # Set or update intent, depending on whether the variable is an index - # in var_local_name_define or the actual parent of that variable. - if not parent_standard_name in self.parents[ccpp_stage].keys(): - self.parents[ccpp_stage][parent_standard_name] = copy.deepcopy(parent_var) - # Copy the intent of the actual variable being processed - if local_name_define == parent_local_name_define: - self.parents[ccpp_stage][parent_standard_name].intent = var.intent - # It's an index for the actual variable being processed --> intent(in) - else: - self.parents[ccpp_stage][parent_standard_name].intent = 'in' - elif self.parents[ccpp_stage][parent_standard_name].intent == 'in': - # Adjust the intent if the actual variable is not intent(in) - if local_name_define == parent_local_name_define and not var.intent == 'in': - self.parents[ccpp_stage][parent_standard_name].intent = 'inout' - # It's an index for the actual variable being processed, intent is ok - #else: - # # nothing to do - elif self.parents[ccpp_stage][parent_standard_name].intent == 'out': - # Adjust the intent if the actual variable is not intent(out) - if local_name_define == parent_local_name_define and not var.intent == 'out': - self.parents[ccpp_stage][parent_standard_name].intent = 'inout' - # Adjust the intent, because the variable is also used as index variable - else: - self.parents[ccpp_stage][parent_standard_name].intent = 'inout' - - # Record the parent information for this variable (with standard name var_standard_name) - if local_name_define == parent_local_name_define: - local_vars[var_standard_name] = { - 'name' : metadata_define[var_standard_name][0].local_name, - 'kind' : metadata_define[var_standard_name][0].kind, - 'parent_standard_name' : parent_standard_name - } - - # Reset parent to actual parent of the variable with standard name var_standard_name - if local_vars[var_standard_name]['parent_standard_name']: - parent_standard_name = local_vars[var_standard_name]['parent_standard_name'] - parent_var = metadata_define[parent_standard_name][0] - - elif local_vars[var_standard_name]['parent_standard_name']: - parent_standard_name = local_vars[var_standard_name]['parent_standard_name'] - parent_var = metadata_define[parent_standard_name][0] - # Update intent information if necessary - if self.parents[ccpp_stage][parent_standard_name].intent == 'in' and not var.intent == 'in': - self.parents[ccpp_stage][parent_standard_name].intent = 'inout' - elif self.parents[ccpp_stage][parent_standard_name].intent == 'out' and not var.intent == 'out': - self.parents[ccpp_stage][parent_standard_name].intent = 'inout' - - # End of iteration (while) loop, increase iteration counter - iteration += 1 - - # Loop over actual arguments for this subroutine and create the argument list. - # This is not required for the additional dimensions and variables. - for var_standard_name in arguments[scheme_name][subroutine_name]: - # Pick the correct variable for this module/scheme/subroutine - # from the list of requested variables - for var in metadata_request[var_standard_name]: - if container == var.container: - break - - # We need some information about the host model variable - (dimensions_target_name, dim_string_target_name) = extract_dimensions_from_local_name(var.target) - - # Derive correct horizontal loop extent for this variable for the rest of this function - if var.rank: - array_size = [] - dim_substrings = [] - for dim in var.dimensions: - - # Work around for GNU compiler bugs related to allocatable strings - # in older versions of GNU (at least 9.2.0) - if var.rank and var.type == 'character': - use_explicit_dimension = False - else: - use_explicit_dimension = True - - # This is not supported/implemented: tmpvar would have one dimension less - # than the original array, and the metadata requesting the variable would - # not pass the initial test that host model variables and scheme variables - # have the same rank. - if dim == CCPP_BLOCK_NUMBER: - raise Exception("{} cannot be part of the dimensions of variable {}".format( - CCPP_BLOCK_NUMBER, var_standard_name)) - else: - # Handle dimensions like "A:B", "A:3", "-1:Z" - if ':' in dim: - dims = [ x.lower() for x in dim.split(':')] - try: - dim0 = int(dims[0]) - dim0 = dims[0] - except ValueError: - if not dims[0].lower() in metadata_define.keys(): - raise Exception('Dimension {}, required by variable {}, not defined in host model metadata'.format( - dims[0].lower(), var_standard_name)) - dim0 = metadata_define[dims[0].lower()][0].local_name - try: - dim1 = int(dims[1]) - dim1 = dims[1] - except ValueError: - # Use correct horizontal variables in run phase - if ccpp_stage == 'run' and dims[1].lower() == CCPP_HORIZONTAL_LOOP_EXTENT: - # Provide backward compatibility with blocked data structures - # and bypass the unresolved problems with inactive data - - # For this, we need to check if the host model variable - # is a contiguous array (it's horizontal dimension is - # CCPP_HORIZONTAL_DIMENSION) or part of a blocked data - # structure (it's horizontal dimension is CCPP_HORIZONTAL_LOOP_EXTENT) - for dim in metadata_define[var_standard_name][0].dimensions: - if ':' in dim: - host_var_dims = [x.lower() for x in dim.split(':')] - # Single dimensions are indices and should not be recorded as a dimension! - else: - raise Exception("THIS SHOULD NOT HAPPEN WITH CAPGEN'S METADATA PARSER") - if CCPP_HORIZONTAL_DIMENSION in host_var_dims: - host_var_is_contiguous = True - elif CCPP_HORIZONTAL_LOOP_EXTENT in host_var_dims: - host_var_is_contiguous = False - if CCPP_HORIZONTAL_LOOP_BEGIN in metadata_define.keys() and host_var_is_contiguous: - dim0 = metadata_define[CCPP_HORIZONTAL_LOOP_BEGIN][0].local_name - dim1 = metadata_define[CCPP_HORIZONTAL_LOOP_END][0].local_name - use_explicit_dimension = True - else: - dim0 = metadata_define[CCPP_CONSTANT_ONE][0].local_name - dim1 = metadata_define[CCPP_HORIZONTAL_LOOP_EXTENT][0].local_name - # Remove this variable so that we can catch errors - # if it doesn't get set even though it should - del host_var_is_contiguous - else: - if not dims[1].lower() in metadata_define.keys(): - raise Exception('Dimension {}, required by variable {}, not defined in host model metadata'.format( - dims[1].lower(), var_standard_name)) - dim1 = metadata_define[dims[1].lower()][0].local_name - # Single dimensions are indices and should not be recorded as a dimension! - else: - raise Exception("THIS SHOULD NOT HAPPEN WITH CAPGEN'S METADATA PARSER") - - # DH* TODO REMOVE THIS ENTIRE BLOCK in a future PR to feature/capgen - # This block should not be needed, the metadata parser should take care - # of flagging invalid dimensions for the host model or the physics. - # TODO: create a test suite to make sure these things are caught - # by the metadata parser - do this on the feature/capgen branch! - if ccpp_stage == 'run': - # This should not happen when parsing metadata with capgen's metadata parser, remove? - if dims[1] == CCPP_HORIZONTAL_LOOP_EXTENT and not dim0: - raise Exception(f"Invalid metadata for scheme {scheme_name}: " + \ - f"horizontal dimension for {var_standard_name} is {var.dimensions}") - # This should not happen when parsing metadata with capgen's metadata parser, remove? - elif CCPP_HORIZONTAL_LOOP_BEGIN in dims or CCPP_HORIZONTAL_LOOP_END in dims or \ - CCPP_HORIZONTAL_DIMENSION in dims: - raise Exception(f"Invalid metadata for scheme {scheme_name}: " + \ - f"horizontal dimension for {var_standard_name} is {var.dimensions}") - else: - # This should not happen when parsing metadata with capgen's metadata parser, remove? - if dims[1] == CCPP_HORIZONTAL_DIMENSION and not dim0: - raise Exception(f"Invalid metadata for scheme {scheme_name}: " + \ - f"horizontal dimension for {var_standard_name} is {var.dimensions}") - # This should not happen when parsing metadata with capgen's metadata parser, remove? - if CCPP_HORIZONTAL_LOOP_BEGIN in dims or CCPP_HORIZONTAL_LOOP_END in dims or \ - CCPP_LOOP_EXTENT in dims: - raise Exception(f"Invalid metadata for scheme {scheme_name}: " + \ - f"horizontal dimension for {var_standard_name} is {var.dimensions}") - # *DH - - # DH* TODO: WE CANNOT ACTIVATE USING EXPLICIT HORIZONTAL DIMENSIONS - # UNTIL WE HAVE SOLVED THE PROBLEM WITH INACTIVE (NON-ALLOCATED) - # ARRAYS. THIS MUST BE ADDRESSED BEFORE WE SWITCH TO CONTIGUOUS - # ARRAYS FOR MODELS LIKE THE UFS-WEATHER-MODEL! - if use_explicit_dimension: - if dim0 == dim1: - array_size.append('1') - dim_substrings.append(f'{dim1}') - else: - array_size.append(f'({dim1}-{dim0}+1)') - dim_substrings.append(f'{dim0}:{dim1}') - else: - if dim0 == dim1: - array_size.append('1') - dim_substrings.append(f':') - else: - array_size.append(f'({dim1}-{dim0}+1)') - dim_substrings.append(f':') - - # Now we need to compare dim_substrings with a possible dim_string_target_name and merge them - if dimensions_target_name: - if len(dimensions_target_name) < len(dim_substrings): - raise Exception("THIS SHOULD NOT HAPPEN") - dim_counter = 0 - # We need two different dim strings for the following. The first, - # called 'dim_string' is used for the incoming variable (host model - # variable) and must contain all dimensions and indices. The second, - # called 'dim_string_allocate' is used for the allocation of temporary - # variables used for unit conversions etc. This 'dim_string_allocate' - # only contains the dimensions, not the indices of the target variable. - # Example: a scheme requests a variable foo that is a slice of a host - # model variable bar: foo(1:n) = bar(1:n,1). If a variable transformation - # is required, mkstatic creates a temporary variable tmpvar of rank 1. - # The allocation and assignment of this variable must then be - # allocate(tmpvar(1:n)) - # tmpvar(1:n) = bar(1:n,1) - # Likewise, when scheme baz is called, the callstring must be - # call baz(...,foo=tmpvar(1:n),...) - # Hence the need for two different dim strings in the following code. - # See also https://github.com/NCAR/ccpp-framework/issues/598. - dim_string = '(' - dim_string_allocate = '(' - for dim in dimensions_target_name: - if ':' in dim: - dim_string += dim_substrings[dim_counter] + ',' - dim_string_allocate += dim_substrings[dim_counter] + ',' - dim_counter += 1 - else: - dim_string += dim + ',' - # Don't add to dim_string_allocate! - dim_string = dim_string.rstrip(',') + ')' - dim_string_allocate = dim_string_allocate.rstrip(',') + ')' - # Consistency check to make sure all dimensions from metadata are 'used' - if dim_counter < len(dim_substrings): - raise Exception(f"Mismatch of derived dimensions from metadata {dim_substrings} " + \ - f"vs target local name {dimensions_target_name} for {var_standard_name} and " + \ - f"scheme {scheme_name} / phase {ccpp_stage}") - else: - dim_string = '({})'.format(','.join(dim_substrings)) - dim_string_allocate = dim_string - var_size_expected = '({})'.format('*'.join(array_size)) - else: - if dimensions_target_name: - dim_string = dim_string_target_name - else: - dim_string = '' - # A scalar variable doesn't get allocated, set to safe value - dim_string_allocate = '' - var_size_expected = 1 - - # To assist debugging efforts, check if arrays have the correct size (ignore scalars for now) - assign_test = '' - if debug: - if ccpp_stage in ['init', 'timestep_init', 'timestep_finalize', 'finalize'] and \ - CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER] in local_vars[var_standard_name]['name'] and \ - '{}:{}'.format(CCPP_CONSTANT_ONE,CCPP_HORIZONTAL_DIMENSION) in var.dimensions: - # We don't need extra tests for blocked arrays, because the de-blocking logic below - # will catch any out of bound reads with the appropriate compiler flags. It naturally - # deals with non-uniform block sizes etc. - pass - # Some older versions of GNU currently in use can not do these variable size tests on strings - # 0x5b6fdd gimplify_expr(tree_node**, gimple**, gimple**, bool (*)(tree_node*), int) - # /tmp/role.apps/spack-stage/spack-stage-gcc-9.2.0-ku6r4f5qa5obpfnqpa6pezhogxq6sp7h/spack-src/gcc/gimplify.c:13477 - elif var.rank and not var.type == 'character': - assign_test = ''' ! Check if variable {var_name} is associated/allocated and has the correct size - if (size({var_name}{dim_string})/={var_size_expected}) then - write({ccpp_errmsg}, '(2(a,i8))') 'Detected size mismatch for variable {var_name}{dim_string} in group {group_name} before {subroutine_name}, expected ', & - {var_size_expected}, ' but got ', size({var_name}{dim_string}) - ierr = 1 - return - end if -'''.format(var_name=local_vars[var_standard_name]['name'].replace(dim_string_target_name, ''), - dim_string=dim_string, - var_size_expected=var_size_expected, - ccpp_errmsg=CCPP_INTERNAL_VARIABLES[CCPP_ERROR_MSG_VARIABLE], group_name = self.name, - subroutine_name=subroutine_name) - # end if debug - - # kind_string is used for automated unit conversions, i.e. foo_kind_phys - kind_string = '_' + local_vars[var_standard_name]['kind'] if local_vars[var_standard_name]['kind'] else '' - - # conditional is the conditional allocation, which can be '.true.', '.false.', or any regular Fortran logical expression - conditional=conditionals[var_standard_name] - - # If the host variable is conditionally allocated, create a pointer for it - if not conditional == '.true.': - # Reuse existing temporary pointer variable, if possible; otherwise add a local pointer (tmpptr) - if local_vars[var_standard_name]['name'] in tmpptrs.keys(): - tmpptr = tmpptrs[local_vars[var_standard_name]['name']] - else: - tmpptr_cnt += 1 - tmpptr = copy.deepcopy(var) - tmpptr.local_name = '{0}_{1}_ptr'.format(var.local_name, tmpptr_cnt) - tmpptr.pointer = True - tmpptrs[local_vars[var_standard_name]['name']] = tmpptr - - # Convert blocked data in init and finalize steps - only required for variables with block number and horizontal_dimension - if ccpp_stage in ['init', 'timestep_init', 'timestep_finalize', 'finalize'] and \ - CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER] in local_vars[var_standard_name]['name'] and \ - '{}:{}'.format(CCPP_CONSTANT_ONE,CCPP_HORIZONTAL_DIMENSION) in var.dimensions: - # Reuse existing temporary variable, if possible; otherwise add a local variable (tmpvar) - if local_vars[var_standard_name]['name'] in tmpvars.keys(): - tmpvar = tmpvars[local_vars[var_standard_name]['name']] - actions_in = tmpvar.actions['in'] - actions_out = tmpvar.actions['out'] - else: - tmpvar_cnt += 1 - tmpvar = copy.deepcopy(var) - tmpvar.local_name = '{0}_{1}_local'.format(var.local_name, tmpvar_cnt) - - # Create string for allocating the temporary array by converting the dimensions - # (in standard_name format) to local names as known to the host model - alloc_dimensions = [] - for dim in tmpvar.dimensions: - # This is not supported/implemented: tmpvar would have one dimension less - # than the original array, and the metadata requesting the variable would - # not pass the initial test that host model variables and scheme variables - # have the same rank. - if dim == CCPP_BLOCK_NUMBER: - raise Exception("{} cannot be part of the dimensions of variable {}".format( - CCPP_BLOCK_NUMBER, var_standard_name)) - else: - # Handle dimensions like "A:B", "A:3", "-1:Z" - if ':' in dim: - dims = [ x.lower() for x in dim.split(':')] - try: - dim0 = int(dims[0]) - except ValueError: - dim0 = metadata_define[dims[0]][0].local_name - try: - dim1 = int(dims[1]) - except ValueError: - dim1 = metadata_define[dims[1]][0].local_name - # Single dimensions - else: - dim0 = 1 - try: - dim1 = int(dim) - except ValueError: - dim1 = metadata_define[dim][0].local_name - alloc_dimensions.append('{}:{}'.format(dim0,dim1)) - - # Padding of additional dimensions - before and after the horizontal dimension - hdim_index = tmpvar.dimensions.index('{}:{}'.format(CCPP_CONSTANT_ONE,CCPP_HORIZONTAL_DIMENSION)) - dimpad_before = '' + ':,'*(len(tmpvar.dimensions[:hdim_index])) - dimpad_after = '' + ',:'*(len(tmpvar.dimensions[hdim_index+1:])) - - # Add necessary local variables for looping over blocks - var_defs_manual.append('integer :: ib, nb') - - # Define actions before. Always copy data in, independent of intent. - # We intentionally omit the dim string for the assignment on the right hand side, - # since it worked without until now, since coding this up together with chunked array - # logic is tricky, and since all this logic will go away after the models transitioned - # to chunked arrays. - actions_in = ''' ! Allocate local variable to copy blocked data {var} into a contiguous array - allocate({tmpvar}({dims})) - ib = 1 - do nb=1,{block_count} - {tmpvar}({dimpad_before}ib:ib+{block_size}-1{dimpad_after}) = {var} - ib = ib+{block_size} - end do -'''.format(tmpvar=tmpvar.local_name, - block_count=metadata_define[CCPP_BLOCK_COUNT][0].local_name.replace(CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER],'nb'), - block_size=metadata_define[CCPP_HORIZONTAL_LOOP_EXTENT][0].local_name.replace(CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER],'nb'), - var=tmpvar.target.replace(CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER],'nb'), - dims=','.join(alloc_dimensions), - dimpad_before=dimpad_before, - dimpad_after=dimpad_after, - ) - # Define actions after, depending on intent. - if var.intent in [ 'inout', 'out' ]: - actions_out = ''' ib = 1 - do nb=1,{block_count} - {var} = {tmpvar}({dimpad_before}ib:ib+{block_size}-1{dimpad_after}) - ib = ib+{block_size} - end do - deallocate({tmpvar}) -'''.format(tmpvar=tmpvar.local_name, - block_count=metadata_define[CCPP_BLOCK_COUNT][0].local_name.replace(CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER],'nb'), - block_size=metadata_define[CCPP_HORIZONTAL_LOOP_EXTENT][0].local_name.replace(CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER],'nb'), - var=tmpvar.target.replace(CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER],'nb'), - dimpad_before=dimpad_before, - dimpad_after=dimpad_after, - ) - else: - actions_out = ''' deallocate({tmpvar}) -'''.format(tmpvar=tmpvar.local_name) - - # Set/update actions for this temporary variable - tmpvar.actions = {'in' : actions_in, 'out' : actions_out} - tmpvars[local_vars[var_standard_name]['name']] = tmpvar - - # Add unit conversions, if necessary - if var.actions['in']: - # Add unit conversion before entering the subroutine, after allocating the temporary - # array holding the non-blocked data and copying the blocked data to it - actions_in = actions_in + \ - ' {t} = {c}\n'.format(t=tmpvar.local_name, - c=var.actions['in'].format(var=tmpvar.local_name, - kind=kind_string)) - - # If the variable is conditionally allocated, assign pointer - if not conditional == '.true.': - # We don't want the dimstring here - this can lead to dimension mismatches. - # We know for sure that we need to reference the entire de-blocked array anyway. - actions_in += ' {p} => {t}\n'.format(p=f"{tmpptr.local_name}_array({CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER]})%p", - t=tmpvar.local_name, d=dim_string) - - if var.actions['out']: - # Add unit conversion after returning from the subroutine, before copying the non-blocked - # data back to the blocked data and deallocating the temporary array - actions_out = ' {t} = {c}\n'.format(t=tmpvar.local_name, - c=var.actions['out'].format(var=tmpvar.local_name, - kind=kind_string)) + \ - actions_out - - # If the variable is conditionally allocated, nullify pointer - if not conditional == '.true.': - actions_out += ' nullify({p})\n'.format(p=f"{tmpptr.local_name}_array({CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER]})%p") - - # Add the conditionals for the "before" operations - actions_before += ''' - if ({conditional}) then -{actions_in} - end if -'''.format(conditional=conditional, actions_in=actions_in.rstrip('\n')) - # Add the conditionals for the "after" operations - actions_after += ''' - if ({conditional}) then -{actions_out} - end if -'''.format(conditional=conditional, actions_out=actions_out.rstrip('\n')) - - # Add to argument list - if conditional == '.true.': - arg = '{local_name}={var_name},'.format(local_name=var.local_name, var_name=tmpvar.local_name) - else: - arg = '{local_name}={ptr_name},'.format(local_name=var.local_name, - ptr_name=f"{tmpptr.local_name}_array({CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER]})%p") - - # Variables stored in blocked data structures but without horizontal dimension not supported at this time (doesn't make sense anyway) - elif ccpp_stage in ['init', 'timestep_init', 'timestep_finalize', 'finalize'] and \ - CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER] in local_vars[var_standard_name]['name']: - raise Exception("Variables stored in blocked data structures but without horizontal dimension not supported in phases ' + \ - 'init, timestep_init, timestep_finalize, finalize at this time: {} in {}".format(var_standard_name, subroutine_name)) - - # Limitations for UFS: Variables stored in threaded data structures (i.e. only for one block at a time) in GFS_interstitial DDT - # are not supported at this time (doesn't make sense anyway) - elif ccpp_stage in ['init', 'timestep_init', 'timestep_finalize', 'finalize'] and \ - CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER] in local_vars[var_standard_name]['name']: - raise Exception("Variables stored in thread-specific data structures (GFS_interstitial DDT) are not supported in phases ' + \ - 'init, timestep_init, timestep_finalize, finalize at this time: {} in {}".format(var_standard_name, subroutine_name)) - - # Unit conversions without converting blocked data structures - elif var.actions['in'] or var.actions['out']: - # If requested, check that arrays are allocated/associated and have the correct size - if debug: - actions_in = assign_test - else: - actions_in = '' - actions_out = '' - if local_vars[var_standard_name]['name'] in tmpvars.keys(): - # If the variable already has a local variable (tmpvar), reuse it - tmpvar = tmpvars[local_vars[var_standard_name]['name']] - else: - # Add a local variable (tmpvar) for this variable - tmpvar_cnt += 1 - tmpvar = copy.deepcopy(var) - tmpvar.local_name = 'tmpvar_{0}'.format(tmpvar_cnt) - tmpvars[local_vars[var_standard_name]['name']] = tmpvar - if tmpvar.rank: - # Add allocate statement if the variable has a rank > 0 using the dimstring derived above - actions_in += f' allocate({tmpvar.local_name}{dim_string_allocate})\n' - if var.actions['in']: - # Add unit conversion before entering the subroutine - actions_in += ' {t} = {c}{d}\n'.format(t=tmpvar.local_name, - c=var.actions['in'].format(var=tmpvar.target.replace(dim_string_target_name, ''), - kind=kind_string), - d=dim_string) - # If the variable is conditionally allocated, assign pointer - if not conditional == '.true.': - actions_in += ' {p} => {t}{d}\n'.format(p=f"{tmpptr.local_name}_array({CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER]})%p", - t=tmpvar.local_name, d=dim_string_allocate) - if var.actions['out']: - # Add unit conversion after returning from the subroutine - actions_out += ' {v}{d} = {c}\n'.format(v=tmpvar.target.replace(dim_string_target_name, ''), - d=dim_string, - c=var.actions['out'].format(var=tmpvar.local_name, - kind=kind_string)) - # If the variable is conditionally allocated, nullify pointer - if not conditional == '.true.': - actions_out += ' nullify({p})\n'.format(p=f"{tmpptr.local_name}_array({CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER]})%p") - - if tmpvar.rank: - # Add deallocate statement if the variable has a rank > 0 - actions_out += ' deallocate({t})\n'.format(t=tmpvar.local_name) - - # Add the conditionals for the "before" operations - actions_before += ''' - if ({conditional}) then -{actions_in} - end if -'''.format(conditional=conditional, actions_in=actions_in.rstrip('\n')) - # Add the conditionals for the "after" operations - actions_after += ''' - if ({conditional}) then -{actions_out} - end if -'''.format(conditional=conditional, actions_out=actions_out.rstrip('\n')) - - # Add to argument list - if conditional == '.true.': - arg = '{local_name}={var_name}{dim_string},'.format(local_name=var.local_name, - var_name=tmpvar.local_name.replace(dim_string_target_name, ''), dim_string=dim_string_allocate) - else: - arg = '{local_name}={ptr_name},'.format(local_name=var.local_name, - ptr_name=f"{tmpptr.local_name}_array({CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER]})%p") - - # Ordinary variables, no blocked data or unit conversions - elif var_standard_name in arguments[scheme_name][subroutine_name]: - if debug and assign_test: - actions_in = assign_test - else: - actions_in = '' - actions_out = '' - # If the variable is conditionally allocated, assign pointer - if not conditional == '.true.': - actions_in += ' {p} => {t}{d}\n'.format(p=f"{tmpptr.local_name}_array({CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER]})%p", - t=var.target.replace(dim_string_target_name, ''), - d=dim_string) - # If the variable is conditionally allocated, nullify pointer - if not conditional == '.true.': - actions_out += ' nullify({p})\n'.format(p=f"{tmpptr.local_name}_array({CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER]})%p") - - if actions_in: - # Add the conditionals for the "before" operations - actions_before += ''' - if ({conditional}) then -{actions_in} - end if -'''.format(conditional=conditional, actions_in=actions_in.rstrip('\n')) - if actions_out: - # Add the conditionals for the "after" operations - actions_after += ''' - if ({conditional}) then -{actions_out} - end if -'''.format(conditional=conditional, actions_out=actions_out.rstrip('\n')) - - # Add to argument list - if conditional == '.true.': - arg = '{local_name}={var_name}{dim_string},'.format(local_name=var.local_name, - var_name=local_vars[var_standard_name]['name'].replace(dim_string_target_name, ''), dim_string=dim_string) - else: - arg = '{local_name}={ptr_name},'.format(local_name=var.local_name, - ptr_name=f"{tmpptr.local_name}_array({CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER]})%p") - - else: - arg = '' - args += arg - length += len(arg) - # Split args so that lines don't get too long - if length > 70 and not var_standard_name == arguments[scheme_name][subroutine_name][-1]: - args += ' &\n ' - length = 0 - args = args.rstrip(',') - subroutine_call = ''' -{actions_before} - - call {subroutine_name}({args}) - -{actions_after} -'''.format(subroutine_name=subroutine_name, args=args, actions_before=actions_before.rstrip('\n'), actions_after=actions_after.rstrip('\n')) - error_check = '''if ({target_name_flag}/=0) then - {target_name_msg} = "An error occured in {subroutine_name}: " // trim({target_name_msg}) - ierr={target_name_flag} - return - end if -'''.format(target_name_flag=ccpp_error_code_target_name, target_name_msg=ccpp_error_msg_target_name, subroutine_name=subroutine_name) - subcycle_body += ''' - {subroutine_call} - {error_check} - '''.format(subroutine_call=subroutine_call, error_check=error_check) - - module_use += ' use {m}, only: {s}\n'.format(m=module_name, s=subroutine_name) - - # If this subcycle calls any schemes, i.e. has any variables registered - # that need to be passed to the group for this stage, then handle the - # subcycle loops by prepending/appending the necessary code to subcycle_body - subcycle_body_prefix = ''' - ! Start of next subcycle -''' - subcycle_body_suffix = '' - if self.parents[ccpp_stage]: - # Set subcycle loop extent - if ccpp_stage == 'run': - subcycle_body_prefix += ''' - ! Set loop extent variable for the following subcycle - {loop_extent_var_name} = {loop_cnt_max} -'''.format(loop_extent_var_name=ccpp_loop_extent_target_name, - loop_cnt_max=subcycle.loop) - else: - subcycle_body_prefix += ''' - ! Set loop extent variable for the following subcycle - {loop_extent_var_name} = 1 -'''.format(loop_extent_var_name=ccpp_loop_extent_target_name) - # Create subcycle (Fortran do loop) if needed - if subcycle.loop > 1 and ccpp_stage == 'run': - subcycle_body_prefix += ''' - associate(cnt => {loop_var_name}) - do cnt=1,{loop_cnt_max}\n\n'''.format(loop_var_name=ccpp_loop_counter_target_name, - loop_cnt_max=subcycle.loop) - subcycle_body_suffix += ''' - end do - end associate -''' - else: - subcycle_body_prefix += ''' - {loop_var_name} = 1\n'''.format(loop_var_name=ccpp_loop_counter_target_name) - - # Add this subcycle's Fortran body to the group body - if subcycle_body: - body += subcycle_body_prefix + subcycle_body + subcycle_body_suffix - - #For the init stage, for the case when the suite doesn't have any schemes with init phases, - #we still need to add the host-supplied ccpp_t variable to the init group caps so that it is - #available for setting the initialized flag for the particular instance being called. Otherwise, - #the initialized_set_block for the init phase tries to reference the unavailable ccpp_t variable. - if (ccpp_stage == 'init' and not self.parents[ccpp_stage]): - ccpp_var.intent = 'in' - self.parents[ccpp_stage].update({ccpp_var.standard_name:ccpp_var}) - - # Get list of arguments, module use statement and variable definitions for this subroutine (=stage for the group) - (self.arguments[ccpp_stage], sub_module_use, sub_var_defs) = create_arguments_module_use_var_defs( - self.parents[ccpp_stage], metadata_define, - tmpvars.values(), tmpptrs.values()) - sub_argument_list = create_argument_list_wrapped(self.arguments[ccpp_stage]) - - # Remove duplicates from additional manual variable definitions - var_defs_manual = list(set(var_defs_manual)) - - # Write cap - shorten certain ccpp_stages to stay under the 63 character limit for Fortran function names - subroutine = self._suite + '_' + self._name + '_' + CCPP_STAGES[ccpp_stage] + '_cap' - self._subroutines.append(subroutine) - # Test and set blocks for initialization status - check that at least - # the mandatory CCPP error handling arguments are present (i.e. there is - # at least one subroutine that gets called from this group), or skip. - if self.arguments[ccpp_stage]: - initialized_test_block = Group.initialized_test_blocks[ccpp_stage].format( - ccpp_var_name = ccpp_var.local_name, - target_name_flag=ccpp_error_code_target_name, - target_name_msg=ccpp_error_msg_target_name, - name=self._name) - else: - initialized_test_block = '' - initialized_set_block = Group.initialized_set_blocks[ccpp_stage].format( - ccpp_var_name = ccpp_var.local_name, - target_name_flag=ccpp_error_code_target_name, - target_name_msg=ccpp_error_msg_target_name, - name=self._name) - # Create subroutine - local_subs += Group.sub.format(subroutine=subroutine, - argument_list=sub_argument_list, - module_use='\n '.join(sub_module_use), - initialized_test_block=initialized_test_block, - initialized_set_block=initialized_set_block, - var_defs='\n '.join(sub_var_defs + var_defs_manual), - body=body) - - # Write output to stdout or file - if (self.filename is not sys.stdout): - filepath = os.path.split(self.filename)[0] - if filepath and not os.path.isdir(filepath): - os.makedirs(filepath) - # If the file exists, write to temporary file first and compare them: - # - if identical, delete the temporary file and keep the existing one - # and set the group cap update flag to false - # - if different, replace existing file with temporary file and set - # the group cap update flag to true (default value) - # If the file does not exist, write the cap an set the flag to true - if os.path.isfile(self.filename): - write_to_test_file = True - test_filename = self.filename + '.test' - f = open(test_filename, 'w') - else: - write_to_test_file = False - f = open(self.filename, 'w') - else: - f = sys.stdout - f.write(Group.header.format(group=self._name, - module=self._module, - module_use=module_use, - subroutines=', &\n '.join(self._subroutines), - num_instances=CCPP_NUM_INSTANCES)) - f.write(local_subs) - f.write(Group.footer.format(module=self._module)) - if (f is not sys.stdout): - f.close() - # See comment above on updating the group cap or not - if write_to_test_file: - if filecmp.cmp(self.filename, test_filename): - # Files are equal, delete the test cap - # and set update flag to False - os.remove(test_filename) - self.update_cap = False - else: - # Files are different, replace existing cap - # with test cap and set flag to True - # Python 3 only: os.replace(test_filename, self.filename) - os.remove(self.filename) - os.rename(test_filename, self.filename) - self.update_cap = True - else: - self.update_cap = True - return - - @property - def name(self): - '''Get the name of the group.''' - return self._name - - @name.setter - def name(self, value): - self._name = value - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - - @property - def update_cap(self): - '''Get the update_cap flag.''' - return self._update_cap - - @update_cap.setter - def update_cap(self, value): - self._update_cap = value - - @property - def init(self): - '''Get the init flag.''' - return self._init - - @init.setter - def init(self, value): - if not type(value) == types.BooleanType: - raise Exception("Invalid type {0} of argument value, boolean expected".format(type(value))) - self._init = value - - @property - def finalize(self): - '''Get the finalize flag.''' - return self._finalize - - @finalize.setter - def finalize(self, value): - if not type(value) == types.BooleanType: - raise Exception("Invalid type {0} of argument value, boolean expected".format(type(value))) - self._finalize = value - - @property - def suite(self): - '''Get the suite name.''' - return self._suite - - @property - def module(self): - '''Get the module name.''' - return self._module - - @property - def subcycles(self): - '''Get the subcycles.''' - return self._subcycles - - @property - def subroutines(self): - '''Get the subroutine names.''' - return self._subroutines - - def print_debug(self): - '''Basic debugging output about the group.''' - print(self._name) - for subcycle in self._subcycles: - subcycle.print_debug() - - @property - def pset(self): - '''Get the unique physics set of this group.''' - return self._pset - - @pset.setter - def pset(self, value): - self._pset = value - - @property - def parents(self): - '''Get the parent variables for the group.''' - return self._parents - - @parents.setter - def parents(self, value): - self._parents = value - - @property - def arguments(self): - '''Get the argument list of the group.''' - return self._arguments - - @arguments.setter - def arguments(self, value): - self._arguments = value - - -class Subcycle(object): - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - self._schemes = None - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - @property - def loop(self): - '''Get the list of loop.''' - return self._loop - - @loop.setter - def loop(self, value): - if not type(value) is int: - raise Exception("Invalid type {0} of argument value, integer expected".format(type(value))) - self._loop = value - - @property - def schemes(self): - '''Get the list of schemes.''' - return self._schemes - - @schemes.setter - def schemes(self, value): - if not type(value) is list: - raise Exception("Invalid type {0} of argument value, list expected".format(type(value))) - self._schemes = value - - def print_debug(self): - '''Basic debugging output about the subcycle.''' - print(self._loop) - for scheme in self._schemes: - print(scheme) - - -############################################################################### -if __name__ == "__main__": - main() diff --git a/scripts/parse_tools/__init__.py b/scripts/parse_tools/__init__.py deleted file mode 100644 index 4d7ec792..00000000 --- a/scripts/parse_tools/__init__.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Public API for the parse_tools library -""" -import sys -import os.path -sys.path.insert(0, os.path.dirname(__file__)) - -# pylint: disable=wrong-import-position -from parse_source import ParseContext, ParseSource -from parse_source import ParseSyntaxError, ParseInternalError -from parse_source import CCPPError, context_string, type_name -from parse_source import unique_standard_name, reset_standard_name_counter -from parse_object import ParseObject -from parse_checkers import check_fortran_id, FORTRAN_ID -from parse_checkers import FORTRAN_DP_RE -from parse_checkers import FORTRAN_SCALAR_REF, FORTRAN_SCALAR_REF_RE -from parse_checkers import check_fortran_ref, check_fortran_literal -from parse_checkers import check_fortran_intrinsic, check_local_name -from parse_checkers import check_diagnostic_id, check_diagnostic_fixed -from parse_checkers import check_fortran_type, check_balanced_paren -from parse_checkers import fortran_list_match -from parse_checkers import registered_fortran_ddt_name -from parse_checkers import register_fortran_ddt_name -from parse_checkers import registered_fortran_ddt_names -from parse_checkers import check_units, check_dimensions, check_cf_standard_name -from parse_checkers import check_default_value, check_valid_values, check_molar_mass -from parse_log import init_log, set_log_level, flush_log -from parse_log import set_log_to_stdout, set_log_to_null -from parse_log import set_log_to_file, verbose -from preprocess import PreprocStack -from xml_tools import find_schema_file, find_schema_version -from xml_tools import read_xml_file, validate_xml_file -from xml_tools import expand_nested_suites, write_xml_file -from fortran_conditional import FORTRAN_CONDITIONAL_REGEX_WORDS, FORTRAN_CONDITIONAL_REGEX -# pylint: enable=wrong-import-position - -__all__ = [ - 'CCPPError', - 'check_balanced_paren', - 'check_cf_standard_name', - 'check_default_value', - 'check_diagnostic_id', - 'check_diagnostic_fixed', - 'check_dimensions', - 'check_fortran_id', - 'check_fortran_intrinsic', - 'check_fortran_literal', - 'check_fortran_ref', - 'check_fortran_type', - 'check_local_name', - 'check_valid_values', - 'check_molar_mass', - 'context_string', - 'expand_nested_suites', - 'find_schema_file', - 'find_schema_version', - 'flush_log', - 'FORTRAN_DP_RE', - 'FORTRAN_ID', - 'FORTRAN_SCALAR_REF', - 'FORTRAN_SCALAR_REF_RE', - 'init_log', - 'ParseContext', - 'ParseInternalError', - 'ParseSource', - 'ParseSyntaxError', - 'ParseObject', - 'PreprocStack', - 'read_xml_file', - 'register_fortran_ddt_name', - 'registered_fortran_ddt_name', - 'registered_fortran_ddt_names', - 'reset_standard_name_counter', - 'set_log_level', - 'set_log_to_file', - 'set_log_to_null', - 'set_log_to_stdout', - 'type_name', - 'unique_standard_name', - 'validate_xml_file', - 'write_xml_file', - 'FORTRAN_CONDITIONAL_REGEX_WORDS', - 'FORTRAN_CONDITIONAL_REGEX' -] diff --git a/scripts/parse_tools/parse_checkers.py b/scripts/parse_tools/parse_checkers.py deleted file mode 100755 index 9a688a13..00000000 --- a/scripts/parse_tools/parse_checkers.py +++ /dev/null @@ -1,1120 +0,0 @@ -#!/usr/bin/env python3 - -"""Helper functions to validate parsed input""" - -# Python library imports -import re -import sys -import os.path -sys.path.insert(0, os.path.dirname(__file__)) -# CCPP framework imports -from parse_source import CCPPError, ParseInternalError - -######################################################################## - -_UNITLESS_REGEX = "1" -_NON_LEADING_ZERO_NUM = r"[1-9]\d*" -_CHAR_WITH_UNDERSCORE = "([a-zA-Z]+_[a-zA-Z]+)+" -_NEGATIVE_NON_LEADING_ZERO_NUM = f"[-]{_NON_LEADING_ZERO_NUM}" -_POSITIVE_NON_LEADING_ZERO_NUM = f"[+]{_NON_LEADING_ZERO_NUM}" -_UNIT_EXPONENT = f"({_NEGATIVE_NON_LEADING_ZERO_NUM}|{_POSITIVE_NON_LEADING_ZERO_NUM}|{_NON_LEADING_ZERO_NUM})" -_UNIT_REGEX = f"[a-zA-Z]+{_UNIT_EXPONENT}?" -_UNITS_REGEX = rf"^({_CHAR_WITH_UNDERSCORE}|{_UNIT_REGEX}(\s{_UNIT_REGEX})*|{_UNITLESS_REGEX})$" -_UNITS_RE = re.compile(_UNITS_REGEX) -_MAX_MOLAR_MASS = 10000.0 - -def check_units(test_val, prop_dict, error): - """Return if a valid unit, otherwise, None - if is True, raise an Exception if is not valid. - >>> check_units('m s-1', None, True) - 'm s-1' - >>> check_units('kg m-3', None, True) - 'kg m-3' - >>> check_units('m2 s-2', None, True) - 'm2 s-2' - >>> check_units('m+2 s-2', None, True) - 'm+2 s-2' - >>> check_units('1', None, True) - '1' - >>> check_units('', None, False) - - >>> check_units('', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '' is not a valid unit - >>> check_units(' ', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '' is not a valid unit - >>> check_units(['foo'], None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: ['foo'] is invalid; not a string - """ - if isinstance(test_val, str): - if _UNITS_RE.match(test_val.strip()) is None: - if error: - raise CCPPError("'{}' is not a valid unit".format(test_val)) - else: - test_val = None - # end if - # end if - else: - if error: - raise CCPPError("'{}' is invalid; not a string".format(test_val)) - else: - test_val = None - # end if - # end if - return test_val - -def check_dimensions(test_val, prop_dict, error, max_len=0): - """Return if a valid dimensions list, otherwise, None - If > 0, each string in must not be longer than - . - if is True, raise an Exception if is not valid. - >>> check_dimensions(["dim1", "dim2name"], None, False) - ['dim1', 'dim2name'] - >>> check_dimensions([":", ":"], None, False) - [':', ':'] - >>> check_dimensions([":", "dim2"], None, False) - [':', 'dim2'] - >>> check_dimensions(["dim1", ":"], None, False) - ['dim1', ':'] - >>> check_dimensions(["8", "::"], None, False) - ['8', '::'] - >>> check_dimensions(['start1:end1', 'start2:end2'], None, False) - ['start1:end1', 'start2:end2'] - >>> check_dimensions(['start1:', 'start2:end2'], None, False) - ['start1:', 'start2:end2'] - >>> check_dimensions(['start1 :end1', 'start2: end2'], None, False) - ['start1 :end1', 'start2: end2'] - >>> check_dimensions(['size(foo)'], None, False) - ['size(foo)'] - >>> check_dimensions(['size(foo,1) '], None, False) - ['size(foo,1) '] - >>> check_dimensions(['size(foo,1'], None, False) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: Invalid dimension component, size(foo,1 - >>> check_dimensions(["dim1", "dim2name"], None, False, max_len=5) - - >>> check_dimensions(["dim1", "dim2name"], None, True, max_len=5) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'dim2name' is too long (> 5 chars) - >>> check_dimensions("hi_mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'hi_mom' is invalid; not a list - >>> check_dimensions(["1:dim1", "dim2name"], None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '1:dim1 is an invalid dimension name; integer dimension indices not supported - >>> check_dimensions(["ccpp_constant_one:1", "dim2name"], None, True) - ['ccpp_constant_one:1', 'dim2name'] - """ - info_msg = None - if not isinstance(test_val, list): - if error: - raise CCPPError("'{}' is invalid; not a list".format(test_val)) - else: - test_val = None - # end if - else: - for item in test_val: - isplit = item.split(':') - # Check for too many colons - if (len(isplit) > 3): - if error: - errmsg = "'{}' is an invalid dimension range" - raise CCPPError(errmsg.format(item)) - else: - test_val = None - # end if - break - # end if - # Check possible dim styles (a, a:b, a:, :b, :, ::, a:b:c, a::c) - tdims = [x.strip() for x in isplit if len(x) > 0] - starts_at_one = False - if len(tdims) > 0 and tdims[0] == 'ccpp_constant_one': - starts_at_one = True - # end if - is_int = False - for tdim in tdims: - # Check numeric value first - try: - is_int = isinstance(int(tdim), int) - # Allow integer dimensions, but not indices - if is_int: - valid = starts_at_one or len(tdims) == 1 - if not valid: - info_msg = 'integer dimension indices not supported' - # end if - else: - valid = False - # end if - except ValueError as ve: - # Not an integer, try a Fortran ID - valid = check_fortran_id(tdim, None, - error, max_len=max_len) is not None - if not valid: - # Check for size entry -- simple check - tcheck = tdim.strip().lower() - if tcheck[0:4] == 'size': - ploc = check_balanced_paren(tdim[4:]) - if -1 in ploc: - emsg = 'Invalid dimension component, {}' - raise CCPPError(emsg.format(tdim)) - else: - valid = tdim - # end if - # end if - # end if - # End try - if not valid: - if error: - if info_msg: - errmsg = f"'{item}' is an invalid dimension name; {info_msg}" - else: - errmsg = f"'{item}' is an invalid dimension name" - # end if - raise CCPPError(errmsg) - else: - test_val = None - # end if - break - # end if - # end for - # end for - # end if - return test_val - -######################################################################## - -# CF_ID is a string representing the regular expression for CF Standard Names -CF_ID = r"(?i)[a-z][a-z0-9_]*" -__CFID_RE = re.compile(CF_ID+r"$") - -def check_cf_standard_name(test_val, prop_dict, error): - """Return if a valid CF Standard Name, otherwise, None - http://cfconventions.org/Data/cf-standard-names/docs/guidelines.html - if is True, raise an Exception if is not valid. - >>> check_cf_standard_name("hi_mom", None, False) - 'hi_mom' - >>> check_cf_standard_name("hi mom", None, False) - - >>> check_cf_standard_name("hi mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'hi_mom' is not a valid CF Standard Name - >>> check_cf_standard_name("", None, False) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: CCPP Standard Name cannot be blank - >>> check_cf_standard_name("_hi_mom", None, False) - - >>> check_cf_standard_name("2pac", None, False) - - >>> check_cf_standard_name("Agood4tranID", None, False) - 'agood4tranid' - >>> check_cf_standard_name("agoodcfid", None, False) - 'agoodcfid' - """ - if len(test_val) == 0: - raise CCPPError("CCPP Standard Name cannot be blank") - else: - match = __CFID_RE.match(test_val) - # end if - if match is None: - if error: - errmsg = "'{}' is not a valid CCPP Standard Name" - raise CCPPError(errmsg.format(test_val)) - else: - test_val = None - # end if - else: - test_val = test_val.lower() - # end if - return test_val - -######################################################################## - -### Fortran-specific parsing helper variables and functions - -######################################################################## - -# FORTRAN_ID is a string representing the regular expression for Fortran names -FORTRAN_ID = r"([A-Za-z][A-Za-z0-9_]*)" -__FID_RE = re.compile(FORTRAN_ID+r"$") -# Note that the scalar array reference expressions below are not really for -# scalar references because a colon can be a placeholder, unlike in Fortran code -__FORTRAN_AID = r"(?:[A-Za-z][A-Za-z0-9_]*)" -__FORT_INT = r"[0-9]+" -__FORT_DIM = r"(?:"+__FORTRAN_AID+r"|[:]|"+__FORT_INT+r")" -__REPEAT_DIM = r"(?:,\s*"+__FORT_DIM+r"\s*)" -__FORTRAN_SCALAR_ARREF = r"[(]\s*("+__FORT_DIM+r"\s*"+__REPEAT_DIM+r"{0,6})[)]" -# FORTRAN_SCALAR_REF: Pattern of a valid Fortran array reference -# NB: Only allows symbols, no expressions and/or function calls -FORTRAN_SCALAR_REF = r"(?:"+FORTRAN_ID+r"\s*"+__FORTRAN_SCALAR_ARREF+r")" -FORTRAN_SCALAR_REF_RE = re.compile(FORTRAN_SCALAR_REF+r"$") -# FORTRAN_FUNCTION_REF: A Fortran function reference -# NB: Currenly does not support function arguments -FORTRAN_FUNCTION_REF = r"(?:"+FORTRAN_ID+r"\s*[(]\s*[)])" -FORTRAN_FUNCTION_REF_RE = re.compile(FORTRAN_FUNCTION_REF) -FORTRAN_INTRINSIC_TYPES = ["integer", "real", "logical", "complex", - "double precision", "character"] -FORTRAN_DP_RE = re.compile(r"(?i)double\s*precision") -FORTRAN_TYPE_RE = re.compile(r"(?i)type\s*\(\s*("+FORTRAN_ID+r")\s*\)") - -_REGISTERED_FORTRAN_DDT_NAMES = ["ccpp_constituent_prop_ptr_t"] - -######################################################################## - -def check_fortran_id(test_val, prop_dict, error, max_len=0): - """Return if a valid Fortran identifier, otherwise, None - If > 0, must not be longer than . - if is True, raise an Exception if is not valid. - >>> check_fortran_id("hi_mom", None, False) - 'hi_mom' - >>> check_fortran_id("hi_mom", None, False, max_len=5) - - >>> check_fortran_id("hi_mom", None, True, max_len=5) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'hi_mom' is too long (> 5 chars) - >>> check_fortran_id("hi mom", None, False) - - >>> check_fortran_id("hi mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'hi_mom' is not a valid Fortran identifier - >>> check_fortran_id("", None, False) - - >>> check_fortran_id("_hi_mom", None, False) - - >>> check_fortran_id("2pac", None, False) - - >>> check_fortran_id("Agood4tranID", None, False) - 'Agood4tranID' - """ - match = __FID_RE.match(test_val) - if match is None: - if error: - raise CCPPError("'{}' is not a valid Fortran identifier".format(test_val)) - else: - test_val = None - # end if - elif (max_len > 0) and (len(test_val) > max_len): - if error: - raise CCPPError("'{}' is too long (> {} chars)".format(test_val, max_len)) - else: - test_val = None - # end if - # end if - return test_val - -######################################################################## - -def fortran_list_match(test_str): - """Check if could be a list of Fortran expressions. - The list must be enclosed in parentheses and separated by commas. - If the list appears okay, return the items (for further checking) - >>> fortran_list_match('(ccpp_constant_one:dim1)') - ['ccpp_constant_one:dim1'] - >>> fortran_list_match('(foo, bar)') - ['foo', 'bar'] - >>> fortran_list_match('()') - [''] - >>> fortran_list_match('(foo, ,)') - - >>> fortran_list_match('foo, bar') - - >>> fortran_list_match('(foo, bar') - - """ - parens, parene = check_balanced_paren(test_str) - if (parens >= 0) and (parene > parens): - litems = [x.strip() for x in test_str[parens+1:parene].split(',')] - if (len(litems) > 1) and (min([len(x) for x in litems]) == 0): - litems = None - # end if - else: - litems = None - # end if - return litems - -######################################################################## - -def check_fortran_ref(test_val, prop_dict, error, max_len=0): - """Return if a valid simple Fortran variable reference, - otherwise, None. A simple Fortran variable reference is defined as - a scalar id or a scalar array reference. - if is True, raise an Exception if is not valid. - >>> FORTRAN_SCALAR_REF_RE.match("foo( bar, baz )").group(1) - 'foo' - >>> FORTRAN_SCALAR_REF_RE.match("foo( bar, baz )").group(2) - 'bar, baz ' - >>> FORTRAN_SCALAR_REF_RE.match("foo( bar, baz )").group(2).split(',')[0].strip() - 'bar' - >>> FORTRAN_SCALAR_REF_RE.match("foo( :, baz )").group(2).split(',')[0].strip() - ':' - >>> FORTRAN_SCALAR_REF_RE.match("foo( bar, baz )").group(2).split(',')[1].strip() - 'baz' - >>> check_fortran_ref("hi_mom", None, False) - 'hi_mom' - >>> check_fortran_ref("hi_mom", None, False, max_len=5) - - >>> check_fortran_ref("hi_mom", None, True, max_len=5) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'hi_mom' is too long (> 5 chars) - >>> check_fortran_ref("hi mom", None, False) - - >>> check_fortran_ref("hi mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'hi_mom' is not a valid Fortran identifier - >>> check_fortran_ref("", None, False) - - >>> check_fortran_ref("_hi_mom", None, False) - - >>> check_fortran_ref("2pac", None, False) - - >>> check_fortran_ref("Agood4tranID", None, False) - 'Agood4tranID' - >>> check_fortran_ref("foo(bar)", None, False) - 'foo(bar)' - >>> check_fortran_ref("foo( bar, baz )", None, False) - 'foo( bar, baz )' - >>> check_fortran_ref("foo( :, baz )", None, False) - 'foo( :, baz )' - >>> check_fortran_ref("foo( bar, )", None, False) - - >>> check_fortran_ref("foo( bar, )", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'foo( bar, )' is not a valid Fortran scalar reference - >>> check_fortran_ref("foo()", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'foo()' is not a valid Fortran scalar reference - >>> check_fortran_ref("foo(bar, bazz)", None, True, max_len=3) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'bazz' is too long (> 3 chars) in foo(bar, bazz) - >>> check_fortran_ref("foo(barr, baz)", None, True, max_len=3) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'bazr' is too long (> 3 chars) in foo(barr, baz) - >>> check_fortran_ref("fooo(bar, baz)", None, True, max_len=3) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'foo' is too long (> 3 chars) in fooo(bar, baz) - """ - idval = check_fortran_id(test_val, prop_dict, False, max_len=max_len) - if idval is None: - match = FORTRAN_SCALAR_REF_RE.match(test_val) - if match is None: - if error: - emsg = "'{}' is not a valid Fortran scalar reference" - raise CCPPError(emsg.format(test_val)) - else: - test_val = None - # end if - elif max_len > 0: - tokens = test_val.strip().rstrip(')').split('(') - tokens = [tokens[0].strip()] + [x.strip() - for x in tokens[1].split(',')] - for token in tokens: - if len(token) > max_len: - if error: - emsg = "'{}' is too long (> {} chars) in {}" - raise CCPPError(emsg.format(token, max_len, test_val)) - else: - test_val = None - break - # end if - # end if - # end for - # end if - # end if - return test_val - -######################################################################## - -def check_local_name(test_val, prop_dict, error, max_len=0): - """Return if a valid simple Fortran variable reference, - or Fortran constant, otherwise, None. - A simple Fortran variable reference is defined as a scalar id or a - scalar array reference. - A constant is only valid if is not None, the 'protected' - property is present and True, and the 'type' property matches the - type of . - if is True, raise an Exception if is not valid. - >>> check_local_name("hi_mom", None, error=False) - 'hi_mom' - >>> check_local_name('122', {'protected':True,'type':'integer'}, error=False) - '122' - >>> check_local_name('122', None, error=False) - - >>> check_local_name('122', {}, error=False) - - >>> check_local_name('122', {'protected':False,'type':'integer'}, error=False) - - >>> check_local_name('122', {'protected':True,'type':'real'}, error=False) - - >>> check_local_name('-122.e4', {'protected':True,'type':'real'}, error=False) - '-122.e4' - >>> check_local_name('-122.', {'protected':True,'type':'real','kind':'kp'}, error=False) - - >>> check_local_name('-122._kp', {'protected':True,'type':'real','kind':'kp'}, error=False) - '-122._kp' - >>> check_local_name('q(:,:,index_of_water_vapor_specific_humidity)', {}, error=False) - 'q(:,:,index_of_water_vapor_specific_humidity)' - """ - valid_val = None - # First check for a constant - if (prop_dict is not None) and ('protected' in prop_dict): - protected = prop_dict['protected'] - else: - protected = False - # end if - if (prop_dict is not None) and ('type' in prop_dict): - vtype = prop_dict['type'] - else: - vtype = "" - # end if - if (prop_dict is not None) and ('kind' in prop_dict): - kind = prop_dict['kind'] - else: - kind = "" - # end if - if protected and vtype and check_fortran_literal(test_val, vtype, kind): - valid_val = test_val - # end if - if valid_val is None: - valid_val = check_fortran_ref(test_val, prop_dict, error, max_len=max_len) - # end if - return valid_val - - -######################################################################## - -def check_fortran_intrinsic(typestr, error=False): - """Return if a valid Fortran intrinsic type, otherwise, None - if is True, raise an Exception if is not valid. - >>> check_fortran_intrinsic("real", error=False) - 'real' - >>> check_fortran_intrinsic("complex") - 'complex' - >>> check_fortran_intrinsic("integer") - 'integer' - >>> check_fortran_intrinsic("InteGer") - 'InteGer' - >>> check_fortran_intrinsic("logical") - 'logical' - >>> check_fortran_intrinsic("character") - 'character' - >>> check_fortran_intrinsic("double precision") - 'double precision' - >>> check_fortran_intrinsic("double precision") - 'double precision' - >>> check_fortran_intrinsic("doubleprecision") - 'doubleprecision' - >>> check_fortran_intrinsic("char", error=True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'char' is not a valid Fortran type - >>> check_fortran_intrinsic("int") - - >>> check_fortran_intrinsic("char", error=False) - - >>> check_fortran_intrinsic("type") - - >>> check_fortran_intrinsic("complex(kind=r8)") - - """ - chk_type = typestr.strip().lower() - match = chk_type in FORTRAN_INTRINSIC_TYPES - if (not match) and (chk_type[0:6] == 'double'): - # Special case for double precision - match = FORTRAN_DP_RE.match(chk_type) is not None - # End if - if not match: - if error: - raise CCPPError("'{}' is not a valid Fortran type".format(typestr)) - else: - typestr = None - # end if - # end if - return typestr - -######################################################################## - -def check_fortran_type(typestr, prop_dict, error): - """Return if a valid Fortran type, otherwise, None - if is True, raise an Exception if is not valid. - >>> check_fortran_type("real", None, False) - 'real' - >>> check_fortran_type("integer", None, False) - 'integer' - >>> check_fortran_type("InteGer", None, False) - 'InteGer' - >>> check_fortran_type("character", None, False) - 'character' - >>> check_fortran_type("double precision", None, False) - 'double precision' - >>> check_fortran_type("double precision", None, False) - 'double precision' - >>> check_fortran_type("doubleprecision", None, False) - 'doubleprecision' - >>> check_fortran_type("complex", None, False) - 'complex' - >>> check_fortran_type("char", {}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'char' is not a valid Fortran type - >>> check_fortran_type("int", None, False) - - >>> check_fortran_type("char", {}, False) - - >>> check_fortran_type("type", None, False) - - >>> check_fortran_type("type", {}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'type' is not a valid derived Fortran type - >>> check_fortran_type("type(hi mom)", {}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'type(hi mom)' is not a valid derived Fortran type - """ - dt = "" - match = check_fortran_intrinsic(typestr, error=False) - if match is None: - match = registered_fortran_ddt_name(typestr) - dt = " derived" - # end if - if match is None: - if error: - emsg = "'{}' is not a valid{} Fortran type" - raise CCPPError(emsg.format(typestr, dt)) - else: - typestr = None - # end if - # end if - return typestr - -######################################################################## - -def check_fortran_literal(value, typestr, kind): - """Return True iff is a valid Fortran literal of type, . - Note: no attempt is made to handle the older D syntax for real literals. - To promote clean coding, real values MUST have a decimal point, however, - this check is not available for the complex type so we just require - the two components to either both be integers or both be reals. - If is not an empty string, it is required to be present (i.e., if - == 'kind_phys', should be of the form, 123.4_kind_phys) - >>> check_fortran_literal("123", "integer", "") - True - >>> check_fortran_literal("123", "INTEGER", "") - True - >>> check_fortran_literal("-123", "integer", "") - True - >>> check_fortran_literal("+123", "integer", "") - True - >>> check_fortran_literal("+123", "integer", "kind_int") - False - >>> check_fortran_literal("+123_kind_int", "integer", "kind_int") - True - >>> check_fortran_literal("+123_int", "integer", "kind_int") - False - >>> check_fortran_literal("123", "real", "") - False - >>> check_fortran_literal("123.", "real", "") - True - >>> check_fortran_literal("123.45", "real", "kind_phys") - False - >>> check_fortran_literal("123.45_8", "real", "kind_phys") - False - >>> check_fortran_literal("123.45_kind_phys", "real", "kind_phys") - True - >>> check_fortran_literal("123", "double precision", "") - False - >>> check_fortran_literal("123.", "doubleprecision", "") - True - >>> check_fortran_literal("123.45", "double precision", "kind_phys") - False - >>> check_fortran_literal("123.45_8", "doubleprecision", "kind_phys") - False - >>> check_fortran_literal("123.45_kp", "doubleprecision", "kp") - True - >>> check_fortran_literal("123", "logical", "") - False - >>> check_fortran_literal(".true.", "logical", "") - True - >>> check_fortran_literal(".false.", "logical", "") - True - >>> check_fortran_literal("T", "logical", "") - False - >>> check_fortran_literal("F", "logical", "") - False - >>> check_fortran_literal(".TRUE.", "logical", "kind_log") - False - >>> check_fortran_literal(".TRUE._kind_log", "logical", "kind_log") - True - >>> check_fortran_literal("(123.,456.)", "complex", "") - True - >>> check_fortran_literal("(123. , 456.)", "complex", "") - True - >>> check_fortran_literal("(123.,456", "complex", "") - False - >>> check_fortran_literal("(123. , 456.)", "complex", "kp") - False - >>> check_fortran_literal("(123._kp , 456)", "complex", "kp") - False - >>> check_fortran_literal("(123._kp , 456._kp)", "complex", "kp") - True - >>> check_fortran_literal("'hi mom'", "character", "") - True - >>> check_fortran_literal("'hi mom", "character", "") - False - >>> check_fortran_literal('"hi mom"', "character", "") - True - >>> check_fortran_literal('"hi""mom"', "character", "") - True - >>> check_fortran_literal('"hi" "mom"', "character", "") - False - >>> check_fortran_literal("'hi''there''mom'", "character", "") - True - >>> check_fortran_literal("'hi mom'", "character", "kc") - False - >>> check_fortran_literal("kc_'hi mom'", "character", "kc") - True - >>> check_fortran_literal("123._kp", "float", "kp") #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: ERROR: 'float' is not a Fortran intrinsic type - """ - valid = True - if FORTRAN_DP_RE.match(typestr.strip()) is not None: - vtype = 'real' - else: - vtype = typestr.lower() - # end if - # Check complex first - if vtype == 'complex': - cvals = value.strip().split(',') - if len(cvals) == 2: - tp = 'integer' - if ('.' in cvals[0]) and ('.' in cvals[1]): - tp = 'real' - elif ('.' in cvals[0]) or ('.' in cvals[1]): - valid = False - # end if - if (cvals[0][0] == '(') and (cvals[1][-1] == ')'): - valid = valid and check_fortran_literal(cvals[0][1:], tp, kind) - valid = valid and check_fortran_literal(cvals[1][:-1], tp, kind) - else: - valid = False - # end if - else: - valid = False - elif valid: - vparts = value.strip().split('_') - if vtype == 'character': - if len(vparts) > 1: - val = vparts[-1] - vkind = '_'.join(vparts[0:-1]) - else: - val = vparts[0] - vkind = '' - # end if - else: - val = vparts[0] - if len(vparts) > 1: - vkind = '_'.join(vparts[1:]) - else: - vkind = '' - # end if - # end if - if vkind != kind.lower(): - valid = False - # end if, kind is okay, check value - if valid and (vtype == 'integer'): - try: - vtest = int(val) - except ValueError as ve: - valid = False - # End try - elif valid and (vtype == 'real'): - if '.' not in val: - valid = False - else: - try: - vtest = float(val) - except ValueError as ve: - valid = False - # End try - # end if - elif valid and (vtype == 'logical'): - valid = (val.upper() == '.TRUE.') or (val.upper() == '.FALSE.') - elif valid and (vtype == 'character'): - sep = val[0] - cparts = val.split(sep) - # We must have balanced delimiters - if len(cparts)%2 == 0: - valid = False - else: - for index in range(len(cparts)): - if (index%2 == 0) and (len(cparts[index]) > 0): - valid = False - break - # end if - # end for - # end if (else okay) - elif valid: - errmsg = "ERROR: '{}' is not a Fortran intrinsic type" - raise ParseInternalError(errmsg.format(typestr)) - # end if (no else) - # end if - return valid - -def check_default_value(test_val, prop_dict, error): - """Return if a valid default value for a CCPP field, - otherwise, None. - If is True, raise an Exception if is not valid. - A valid value is determined by the 'type' of the variable. It is an - error for there to be no 'type' property in . - >>> check_default_value('314', {'type':'integer'}, False) - '314' - >>> check_default_value('314', {'type':'integer'}, True) - '314' - >>> check_default_value('314', {'type':'integer', 'kind':'ikind'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 314 is not a valid Fortran integer of kind, ikind - >>> check_default_value('314_ikind', {'type':'integer', 'kind':'ikind'}, True) - '314_ikind' - >>> check_default_value('314', {'type':'real'}, False) - - >>> check_default_value('314', {'type':'real'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 314 is not a valid Fortran real - >>> check_default_value('3.14', {'type':'real'}, False) - '3.14' - >>> check_default_value('314', {'tipe':'integer'}, False) - - >>> check_default_value('314', {'local_name':'foo'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: foo does not have a 'type' attribute - >>> check_default_value('314', {'tipe':'integer'}, False) - - >>> check_default_value('314', None, True) - '314' - """ - valid = None - if prop_dict and ('type' in prop_dict): - valid = test_val - var_type = prop_dict['type'].lower().strip() - if 'kind' in prop_dict: - vkind = prop_dict['kind'].lower().strip() - else: - vkind = '' - # end if - if not check_fortran_literal(test_val, var_type, vkind): - valid = None - if error: - emsg = '{} is not a valid Fortran {}' - if vkind: - emsg += ' of kind, {}' - raise CCPPError(emsg.format(test_val, var_type, vkind)) - # end if - # end if (no else, is okay) - elif prop_dict is None: - # Special case for checks during parsing, always pass - valid = test_val - elif error: - emsg = "{} does not have a 'type' attribute" - if 'local_name' in prop_dict: - lname = prop_dict['local_name'] - else: - lname = 'UNKNOWN' - # end if - raise CCPPError(emsg.format(lname)) - # end if - return valid - -def check_valid_values(test_val, prop_dict, error): - """Return if a valid 'valid_values' attribute value, - otherwise, None. - If is True, raise an Exception if is not valid. - """ - raise ParseInternalError("NOT IMPLEMENTED") - -def check_diagnostic_fixed(test_val, prop_dict, error): - """Return if a valid descriptor for a CCPP diagnostic, - otherwise, None. - If is True, raise an Exception if is not valid. - A fixed diagnostic name is any Fortran identifier, however, it is - an error to specify both 'diagnostic_name' and 'diagnostic_name_fixed'. - >>> check_diagnostic_fixed("foo", {'diagnostic_name_fixed' : 'foo'}, False) - 'foo' - >>> check_diagnostic_fixed("foo", {'diagnostic_name_fixed' : 'foo'}, True) - 'foo' - >>> check_diagnostic_fixed("foo", {'diagnostic_name' : 'foo'}, False) - - >>> check_diagnostic_fixed("foo", {'diagnostic_name':'','local_name':'hi','standard_name':'mom'}, True) - 'foo' - >>> check_diagnostic_fixed("foo", {'diagnostic_name':'foo','local_name':'hi','standard_name':'mom'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: hi (mom) cannot have both 'diagnostic_name' and 'diagnostic_name_fixed' attributes - >>> check_diagnostic_fixed("2foo", {'diagnostic_name_fixed':'foo','local_name':'hi','standard_name':'mom'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '2foo' (hi) is not a valid fixed diagnostic name - """ - valid = test_val - if (prop_dict and ('diagnostic_name' in prop_dict) and - prop_dict['diagnostic_name']): - valid = None - if error: - emsg = "{} ({}) cannot have both 'diagnostic_name' and " - emsg += "'diagnostic_name_fixed' attributes" - if 'local_name' in prop_dict: - lname = prop_dict['local_name'] - else: - lname = 'UNKNOWN' - # end if - if 'standard_name' in prop_dict: - sname = prop_dict['standard_name'] - else: - sname = 'UNKNOWN' - # end if - raise CCPPError(emsg.format(lname, sname)) - # end if - elif check_fortran_id(test_val, prop_dict, False) is None: - valid = None - if error: - emsg = "'{}' ({}) is not a valid fixed diagnostic name" - if 'local_name' in prop_dict: - lname = prop_dict['local_name'] - else: - lname = 'UNKNOWN' - # end if - raise CCPPError(emsg.format(test_val, lname)) - # end if - # end if - return valid - -######################################################################## - -_DIAG_PRE = r"("+FORTRAN_ID+")?" -_DIAG_SUFF = r"([_0-9A-Za-z]+)?" -_DIAG_PROP = r"((\${process}|\${scheme_name})"+_DIAG_SUFF+r")" -_DIAG_RE = re.compile(_DIAG_PRE+_DIAG_PROP+r"?$") - -def check_diagnostic_id(test_val, prop_dict, error): - """Return if a valid descriptor for a CCPP diagnostic, - otherwise, None. - If is True, raise an Exception if is not valid. - A diagnostic name is a Fortran identifier with the optional - addition of one variable substitution. - A variable substitution is a substring of the form of either: - ${process}: The scheme process name will be substituted for this - substring. If this substring is included, it is an error for - there to be no process specified by the scheme (although this - error cannot be detected by this routine). - ${scheme_name}: The scheme name will be substituted for this substring. - It is an error to specify both 'diagnostic_name' and - 'diagnostic_name_fixed'. - >>> check_diagnostic_id("foo", {'diagnostic_name' : 'foo'}, False) - 'foo' - >>> check_diagnostic_id("foo", {'diagnostic_name' : 'foo'}, True) - 'foo' - >>> check_diagnostic_id("foo", {'diagnostic_name_fixed' : 'foo'}, False) - - >>> check_diagnostic_id("foo_${process}", {}, False) - 'foo_${process}' - >>> check_diagnostic_id("foo_${process}_2bad", {}, False) - 'foo_${process}_2bad' - >>> check_diagnostic_id("${process}_2bad", {}, False) - '${process}_2bad' - >>> check_diagnostic_id("foo_${scheme_name}", {}, False) - 'foo_${scheme_name}' - >>> check_diagnostic_id("foo_${scheme_name}_2bad", {}, False) - 'foo_${scheme_name}_2bad' - >>> check_diagnostic_id("${scheme_name}_suff", {}, False) - '${scheme_name}_suff' - >>> check_diagnostic_id("pref_${scheme}_suff", {}, False) - - >>> check_diagnostic_id("pref_${scheme_name_suff", {}, False) - - >>> check_diagnostic_id("pref_$scheme_name}_suff", {}, False) - - >>> check_diagnostic_id("pref_{scheme_name}_suff", {}, False) - - >>> check_diagnostic_id("foo", {'diagnostic_name_fixed':'','local_name':'hi','standard_name':'mom'}, True) - 'foo' - >>> check_diagnostic_id("foo", {'diagnostic_name_fixed':'foo','local_name':'hi','standard_name':'mom'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: hi (mom) cannot have both 'diagnostic_name' and 'diagnostic_name_fixed' attributes - >>> check_diagnostic_id("2foo", {'diagnostic_name':'foo','local_name':'hi','standard_name':'mom'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '2foo' (hi) is not a valid diagnostic name - """ - if (prop_dict and ('diagnostic_name_fixed' in prop_dict) and - prop_dict['diagnostic_name_fixed']): - valid = None - if error: - emsg = "{} ({}) cannot have both 'diagnostic_name' and " - emsg += "'diagnostic_name_fixed' attributes" - if 'local_name' in prop_dict: - lname = prop_dict['local_name'] - else: - lname = 'UNKNOWN' - # end if - if 'standard_name' in prop_dict: - sname = prop_dict['standard_name'] - else: - sname = 'UNKNOWN' - # end if - raise CCPPError(emsg.format(lname, sname)) - # end if - else: - match = _DIAG_RE.match(test_val) - if match is None: - valid = None - if error: - emsg = "'{}' is not a valid diagnostic_name value" - raise CCPPError(emsg.format(test_val)) - # end if - else: - valid = test_val - # end if - # end if - return valid - -######################################################################## - -def check_molar_mass(test_val, prop_dict, error): - """Return if valid molar mass, otherwise, None - if is True, raise an Exception if is not valid. - >>> check_molar_mass('1', None, True) - 1.0 - >>> check_molar_mass('1.0', None, True) - 1.0 - >>> check_molar_mass('1.0', None, False) - 1.0 - >>> check_molar_mass('-1', None, False) - - >>> check_molar_mass('-1.0', None, False) - - >>> check_molar_mass('string', None, False) - - >>> check_molar_mass(10001, None, False) - - >>> check_molar_mass('-1', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '-1' is not a valid molar mass - >>> check_molar_mass('-1.0', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '-1.0' is not a valid molar mass - >>> check_molar_mass('string', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '-1.0' is not a valid molar mass - >>> check_molar_mass(10001, None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '10001' is not a valid molar mass - """ - # Check if input value is an int or float - try: - test_val = float(test_val) - if test_val < 0.0 or test_val > _MAX_MOLAR_MASS: - if error: - raise CCPPError(f"{test_val} is not a valid molar mass") - else: - test_val = None - # end if - # end if - except: - # not an int or float, conditionally throw error - if error: - raise CCPPError(f"{test_val} is invalid; not a float or int") - else: - test_val=None - # end if - # end try - return test_val - -######################################################################## - -def check_balanced_paren(string, start=0, error=False): - """Return indices delineating a balance set of parentheses. - Parentheses in character context do not count. - Left parenthesis search begins at . - Return start and end indices if found - If no parentheses are found, return (-1, -1). - If a left parenthesis is found but no balancing right, return (begin, -1) - where begin is the index where the left parenthesis was found. - If error is True, raise a CCPPError. - >>> check_balanced_paren("foo") - (-1, -1) - >>> check_balanced_paren("(foo, bar)") - (0, 9) - >>> check_balanced_paren("( (foo, bar) )", start=1) - (2, 11) - >>> check_balanced_paren("(size(foo,1), qux)") - (0, 17) - >>> check_balanced_paren("(foo('bar()'))") - (0, 13) - >>> check_balanced_paren("(foo('bar()')") - (0, -1) - >>> check_balanced_paren("(foo('bar()')", error=True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: ERROR: Unbalanced parenthesis in '(foo('bar()')' - """ - index = start - begin = -1 - end = -1 - depth = 0 - inchar = None - str_len = len(string) - while index < str_len: - if (string[index] == '"') or (string[index] == "'"): - if inchar == string[index]: - inchar = None - elif inchar is None: - inchar = string[index] - # else in character context, keep going - # end if - elif inchar is not None: - # In character context, keep going - pass - elif string[index] == '(': - if depth == 0: - begin = index - # end if - depth = depth + 1 - if depth == 0: - break - # end if - elif string[index] == ')': - depth = depth - 1 - if depth == 0: - end = index - break - # end if - # else just keep going - # end if - index = index + 1 - # End while - if (begin >= 0) and (end < 0) and error: - raise CCPPError("ERROR: Unbalanced parenthesis in '{}'".format(string)) - # end if - return begin, end - -######################################################################## - -def registered_fortran_ddt_names(): - return _REGISTERED_FORTRAN_DDT_NAMES - -######################################################################## - -def registered_fortran_ddt_name(name): - if name in _REGISTERED_FORTRAN_DDT_NAMES: - return name - else: - return None - -######################################################################## - -def register_fortran_ddt_name(name): - if name not in _REGISTERED_FORTRAN_DDT_NAMES: - _REGISTERED_FORTRAN_DDT_NAMES.append(name) - -######################################################################## - -if __name__ == "__main__": - # pylint: disable=ungrouped-imports - import doctest - # pylint: enable=ungrouped-imports - fail, _ = doctest.testmod() - sys.exit(fail) -# end if diff --git a/scripts/parse_tools/parse_log.py b/scripts/parse_tools/parse_log.py deleted file mode 100644 index f85a5d09..00000000 --- a/scripts/parse_tools/parse_log.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 - -"""Shared logger for parse processes""" - -# Python library imports -import logging -# CCPP framework imports - -def init_log(name, level=None): - """Initialize a new logger object""" - logger = logging.getLogger(name) - # Turn logging to WARNING if not set - llevel = logger.getEffectiveLevel() - if (level is None) and (llevel == logging.NOTSET): - logger.setLevel(logging.WARNING) - elif level: - logger.setLevel(level) - # End if - set_log_to_stdout(logger) - return logger - -def set_log_level(logger, level): - """Set the logging level of to """ - logger.setLevel(level) - -def remove_handlers(logger): - """Remove all handlers from """ - for handler in list(logger.handlers): - logger.removeHandler(handler) - -def set_log_to_stdout(logger): - """Set to log to standard out""" - remove_handlers(logger) - logger.addHandler(logging.StreamHandler()) - -def set_log_to_null(logger): - """Set to log to NULL""" - remove_handlers(logger) - logger.addHandler(logging.NullHandler()) - -def set_log_to_file(logger, filename): - """Set to log to """ - remove_handlers(logger) - logger.addHandler(logging.StreamHandler()) - -def flush_log(logger): - """Flush all pending output from """ - for handler in list(logger.handlers): - handler.flush() - -def verbose(logger): - """Return true if debug is enabled for this logger""" - return logger.isEnabledFor(logging.DEBUG) diff --git a/scripts/parse_tools/parse_object.py b/scripts/parse_tools/parse_object.py deleted file mode 100644 index 2c5f72f5..00000000 --- a/scripts/parse_tools/parse_object.py +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env python3 -"""A module for the base, ParseObject class""" - -# CCPP framework imports -from parse_source import ParseContext, CCPPError, context_string - -######################################################################## - -class ParseObject(ParseContext): - """ParseObject is a simple class that keeps track of an object's - place in a file and safely produces lines from an array of lines - >>> ParseObject('foobar.F90', []) #doctest: +ELLIPSIS - - >>> ParseObject('foobar.F90', []).filename - 'foobar.F90' - >>> ParseObject('foobar.F90', ["##hi mom",], line_start=1).curr_line() - (None, 1) - >>> ParseObject('foobar.F90', ["first line","## hi mom"], line_start=1).curr_line() - ('## hi mom', 1) - >>> ParseObject('foobar.F90', ["##hi mom",], line_start=1).next_line() - (None, 1) - >>> ParseObject('foobar.F90', ["##first line","## hi mom"], line_start=1).next_line() - ('## hi mom', 1) - >>> ParseObject('foobar.F90', ["## hi \\\\","mom"], line_start=0).next_line() - ('## hi mom', 0) - >>> ParseObject('foobar.F90', ["line1","##line2","## hi mom"], line_start=2).next_line() - ('## hi mom', 2) - >>> ParseObject('foobar.F90', ["## hi \\\\","there \\\\","mom"], line_start=0).next_line() - ('## hi there mom', 0) - >>> ParseObject('foobar.F90', ["!! line1","!! hi mom"], line_start=1).next_line() - ('!! hi mom', 1) - """ - - _max_errors = 32 - - def __init__(self, filename, lines_in, line_start=0): - """Initialize this ParseObject""" - self.__lines = lines_in - self.__line_start = line_start - self.__line_end = line_start - self.__line_next = line_start - self.__num_lines = len(self.__lines) - self.__error_message = "" - self.__num_errors = 0 - super().__init__(linenum=line_start, filename=filename) - - @property - def first_line_num(self): - """Return the first line parsed""" - return self.__line_start - - @property - def last_line_num(self): - """Return the last line parsed""" - return self.__line_end - - def valid_line(self): - """Return True if the current line is valid""" - return (self.line_num >= 0) and (self.line_num < self.__num_lines) - - @property - def error_message(self): - """Return this object's error message""" - return self.__error_message - - def curr_line(self): - """Return the current line (if valid) and the current line number. - If the current line is invalid, return None""" - valid_line = self.valid_line() - _curr_line = None - _my_curr_lineno = self.line_num - if valid_line: - try: - _curr_line = self.__lines[self.line_num].rstrip() - self.__line_next = self.line_num + 1 - self.__line_end = self.__line_next - except CCPPError: - self.add_syntax_err("line", self.line_num) - valid_line = False - # end if - # We allow continuation self.__lines (ending with a single backslash) - if valid_line and _curr_line.endswith('\\'): - next_line, _ = self.next_line() - if next_line is None: - # We ran out of lines, just strip the backslash - _curr_line = _curr_line[0:len(_curr_line)-1] - else: - _curr_line = _curr_line[0:len(_curr_line)-1] + next_line - # end if - # end if - # curr_line should not change the line number - self.line_num = _my_curr_lineno - return _curr_line, self.line_num - - def next_line(self): - """Return the next line in our file (if valid) and the next line's - line number. If the next line is not valid, return None""" - self.line_num = self.__line_next - return self.curr_line() - - def peek_line(self, line_num): - """Return the text of without advancing to that line. - if is out of bounds, return None.""" - if (line_num >= 0) and (line_num < len(self.__lines)): - return self.__lines[line_num] - # end if - return None - - def add_syntax_err(self, token_type, token=None, skip_context=False): - """Add a ParseSyntaxError-type message to this object's error - log, separating it from any previous messages with a newline.""" - if self.__error_message: - if self.__num_errors == self._max_errors: - self.__error_message += '\nMaximum number of errors exceeded' - self.line_num = self.__num_lines # Intentionally walk off end - self.__line_next = self.line_num - elif self.__num_errors > self._max_errors: - # Oops, something went wrong, panic! - raise CCPPError(self.error_message) - # end if - self.__error_message += '\n' - # end if - if self.__num_errors < self._max_errors: - if skip_context: - cstr = "" - else: - cstr = context_string(self) - # end if - if token is None: - self.__error_message += "{}{}".format(token_type, cstr) - else: - self.__error_message += "Invalid {}, '{}'{}".format(token_type, - token, cstr) - # end if - # end if - self.__num_errors += 1 - - def reset_pos(self, line_start=0): - """Attempt to set the current file position to . - If is out of bounds, raise an exception.""" - if (line_start < 0) or (line_start >= self.__num_lines): - emsg = 'Attempt to reset_pos to non-existent line, {}' - raise CCPPError(emsg.format(line_start)) - # end if - self.line_num = line_start - self.__line_next = line_start - - def write_line(self, line_num, line): - """Overwrite line, with . - If is out of bounds, raise an exception.""" - if (line_num < 0) or (line_num >= len(self.__lines)): - emsg = 'Attempt to write non-existent line, {}' - raise CCPPError(emsg.format(line_num)) - # end if - self.__lines[line_num] = line - - def __del__(self): - """Attempt to cleanup memory used by this object""" - try: - del self.__lines - del self.regions - except Exception: - pass # Python does not guarantee much about __del__ conditions - # end try - -######################################################################## diff --git a/scripts/parse_tools/parse_source.py b/scripts/parse_tools/parse_source.py deleted file mode 100644 index 1a4082cb..00000000 --- a/scripts/parse_tools/parse_source.py +++ /dev/null @@ -1,411 +0,0 @@ -#!/usr/bin/env python3 - -"""Classes to aid the parsing process""" - - -# Python library imports -from collections.abc import Iterable -# end if -import copy -import sys -import os.path -import logging -# CCPP framework imports - -class _StdNameCounter(): - """Class to hold a global counter to avoid using global keyword""" - __SNAME_NUM = 0 # Counter for unique standard names - - @classmethod - def new_stdname_number(cls): - """Increment and return the global counter.""" - _StdNameCounter.__SNAME_NUM += 1 - return _StdNameCounter.__SNAME_NUM - - @classmethod - def reset_stdname_counter(cls, reset_val=0): - """Reset the global counter to """ - _StdNameCounter.__SNAME_NUM = reset_val - -############################################################################### -def unique_standard_name(): -############################################################################### - """ - Return a unique standard name. - """ - return 'enter_standard_name_{}'.format(_StdNameCounter.new_stdname_number()) - -############################################################################### -def reset_standard_name_counter(): -############################################################################### - """ - Reset the unique_standard_name counter so that future calls to - unique_standard name will restart. - """ - _StdNameCounter.reset_stdname_counter() - -############################################################################### -def context_string(context=None, with_comma=True, nodir=False): -############################################################################### - """Return a context string if is not None otherwise, return - an empty string. - if with_comma is True, prepend string with ', at ' or ', in '. - >>> context_string() - '' - >>> context_string(with_comma=True) - '' - >>> context_string(context= ParseContext(linenum=32, filename="dir/source.F90"), with_comma=False) - 'dir/source.F90:33' - >>> context_string(context= ParseContext(linenum=32, filename="dir/source.F90"), with_comma=True) - ', at dir/source.F90:33' - >>> context_string(context= ParseContext(linenum=32, filename="dir/source.F90")) - ', at dir/source.F90:33' - >>> context_string(context= ParseContext(filename="dir/source.F90"), with_comma=False) - 'dir/source.F90' - >>> context_string(context= ParseContext(filename="dir/source.F90"), with_comma=True) - ', in dir/source.F90' - >>> context_string(context= ParseContext(filename="dir/source.F90")) - ', in dir/source.F90' - >>> context_string(nodir=True) - '' - >>> context_string(with_comma=True, nodir=True) - '' - >>> context_string(context= ParseContext(linenum=32, filename="dir/source.F90"), with_comma=False, nodir=True) - 'source.F90:33' - >>> context_string(context= ParseContext(linenum=32, filename="dir/source.F90"), with_comma=True, nodir=True) - ', at source.F90:33' - >>> context_string(context= ParseContext(linenum=32, filename="dir/source.F90"), nodir=True) - ', at source.F90:33' - >>> context_string(context= ParseContext(filename="dir/source.F90"), with_comma=False, nodir=True) - 'source.F90' - >>> context_string(context= ParseContext(filename="dir/source.F90"), with_comma=True, nodir=True) - ', in source.F90' - >>> context_string(context= ParseContext(filename="dir/source.F90"), nodir=True) - ', in source.F90' - """ - if context is None: - where_str = '' - elif context.line_num < 0: - where_str = 'in ' - else: - where_str = 'at ' - # End if - if (context is not None) and with_comma: - comma = ', ' - else: - comma = '' - where_str = '' # Override previous setting - # End if - if context is None: - spec = '' - elif nodir: - spec = '{ctx:nodir}' - else: - spec = '{ctx}' - # End if - if context is None: - cstr = "" - else: - cstr = '{comma}{where_str}' + spec - # End if - return cstr.format(comma=comma, where_str=where_str, ctx=context) - -############################################################################### -def type_name(obj): -############################################################################### - """Return the name of the type of """ - return type(obj).__name__ - -############################################################################### -class CCPPError(ValueError): - """Class so programs can log user errors without backtrace""" - def __init__(self, message): - """Initialize this exception""" - logging.shutdown() - super(CCPPError, self).__init__(message) - -######################################################################## - -class ParseSyntaxError(CCPPError): - """Exception that is aware of parsing context""" - def __init__(self, token_type, token=None, context=None): - """Initialize this exception""" - logging.shutdown() - cstr = context_string(context) - if token is None: - message = "{}{}".format(token_type, cstr) - else: - message = "Invalid {}, '{}'{}".format(token_type, token, cstr) - # End if - super(ParseSyntaxError, self).__init__(message) - -######################################################################## - -class ParseInternalError(Exception): - """Exception for internal parser use errors - Note that this error will not be trapped by programs such as ccpp_capgen - """ - def __init__(self, errmsg, context=None): - """Initialize this exception""" - logging.shutdown() - message = "{}{}".format(errmsg, context_string(context)) - super(ParseInternalError, self).__init__(message) - -######################################################################## - -class ParseContextError(CCPPError): - """Exception for errors using ParseContext""" - def __init__(self, errmsg, context): - """Initialize this exception""" - logging.shutdown() - message = "{}{}".format(errmsg, context_string(context)) - super(ParseContextError, self).__init__(message) - -######################################################################## - -class ContextRegion(Iterable): - """Class to imitate the LIFO nature of program language blocks""" - - def __init__(self): - """Initialize this ContextRegion""" - self._lifo = list() - - def push(self, rtype, rname): - """Push a new region onto the stack""" - self._lifo.append([rtype, rname]) - - def pop(self): - """Remove the top item from the stack""" - return self._lifo.pop() - - def type_list(self): - """Return just the types in the list""" - return [x[0] for x in self._lifo] - - def __iter__(self): - """Local version of iterator""" - for item in self._lifo: - yield item[0] - - def __len__(self): - """Local implementation of len builtin""" - return len(self._lifo) - - def __getitem__(self, index): - """Special item getter for a ContextRegion""" - return self._lifo[index] - -######################################################################## - -class ParseContext(): - """A class for keeping track of a parsing position - >>> ParseContext(32, "source.F90") #doctest: +ELLIPSIS - - >>> ParseContext("source.F90", 32) - Traceback (most recent call last): - parse_tools.parse_source.CCPPError: ParseContext linenum must be an int - >>> ParseContext(32, 90) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: ParseContext filenum must be a string - >>> "{}".format(ParseContext(32, "source.F90")) - 'source.F90:33' - >>> "{}".format(ParseContext()) - '' - >>> ParseContext(linenum=32, filename="source.F90").increment(13) - - """ - - def __init__(self, linenum=None, filename=None, context=None): - """Initialize this ParseContext""" - # Set regions first in case of exception - if context is not None: - self.__regions = copy.deepcopy(context.regions) - else: - self.__regions = ContextRegion() - # End if - if context is not None: - # If context is passed, ignore linenum - linenum = context.line_num - elif linenum is None: - linenum = -1 - elif not isinstance(linenum, int): - raise CCPPError('ParseContext linenum must be an int') - # No else, everything is okay - # End if - if context is not None: - # If context is passed, ignore filename - filename = context.filename - elif filename is None: - filename = "" - elif not isinstance(filename, str): - raise CCPPError('ParseContext filename must be a string') - # No else, everything is okay - # End if - self.__linenum = linenum - self.__filename = filename - - def default_module_name(self): - """Return a default module for this file""" - return os.path.splitext(os.path.basename(self.filename))[0] - - @property - def line_num(self): - """Return the current line""" - return self.__linenum - - @line_num.setter - def line_num(self, newnum): - """Set a new line number for this context""" - self.__linenum = newnum - - @property - def filename(self): - """Return the object's filename""" - return self.__filename - - @property - def regions(self): - """Return the object's region list""" - return self.__regions - - def __format__(self, spec): - """Return a string representing the location in a file - Note that self.__linenum is zero based. - can be 'dir' (show filename directory) or 'nodir' filename only. - Any other spec entry is ignored. - """ - if spec == 'dir': - fname = self.__filename - elif spec == 'nodir': - fname = os.path.basename(self.__filename) - else: - fname = self.__filename - # End if - if self.__linenum >= 0: - fmt_str = "{}:{}".format(fname, self.__linenum+1) - else: - fmt_str = "{}".format(fname) - # End if - return fmt_str - - def __str__(self): - """Return a string representing the location in a file - Note that self.__linenum is zero based. - """ - if self.__linenum >= 0: - retstr = "{}:{}".format(self.__filename, self.__linenum+1) - else: - retstr = "{}".format(self.__filename) - # End if - return retstr - - def increment(self, inc=1): - """Increment the location within a file""" - if self.__linenum < 0: - self.__linenum = 0 - # End if - self.__linenum = self.__linenum + inc - - def enter_region(self, region_type, region_name=None, nested_ok=True): - """Mark the entry of a region (e.g., DDT, module, function). - If nested_ok == False, throw an exception if the context is already - inside a region with the same type.""" - if (region_type not in self.__regions.type_list()) or nested_ok: - self.__regions.push(region_type, region_name) - else: - emsg = "Cannot enter a nested {} region" - raise ParseContextError(emsg.format(region_type), self) - # End if - - def leave_region(self, region_type, region_name=None): - """Mark the exit from a region. Check region name if possible""" - if self.__regions: - curr_type, curr_name = self.__regions.pop() - if curr_type != region_type: - emsg = "Trying to exit {} region while currently in {} region" - raise ParseContextError(emsg.format(region_type, curr_type), - self) - # End if - if (region_name is not None) and (curr_name is not None): - if region_name != curr_name: - emsg = "Trying to exit {} {} while currently in {} {}" - raise ParseContextError(emsg.format(region_type, - region_name, - curr_type, - curr_name), self) - # End if - elif (region_name is not None) and (curr_name is None): - emsg = "Trying to exit {} {} while currently in unnamed {} region" - raise ParseContextError(emsg.format(region_type, region_name, - curr_type), self) - # End if - else: - raise ParseContextError("Cannot exit, not currently in any region", - self) - # End if - - def curr_region(self): - """Return the innermost current region""" - curr = None - if self.__regions: - curr = self.__regions[-1] - # No else, will return None - # End if - return curr - - def in_region(self, region_type, region_name=None): - """Return True iff we are currently in """ - return self.curr_region() == [region_type, region_name] - - def region_str(self): - """Create a string describing the current region""" - rgn_str = "" - for index in len(self.__regions): - rtype, rname = self.__regions[index] - if rgn_str: - rgn_str += " ==> " - # End if - rgn_str += "{}".format(rtype) - if rname is not None: - rgn_str += " {}".format(rname) - # End if - # End for - return rgn_str - -######################################################################## - -class ParseSource(): - """ - A simple object for providing source information - >>> ParseSource("myname", "mytype", ParseContext(13, "foo.F90")) #doctest: +ELLIPSIS - - >>> ParseSource("myname", "mytype", ParseContext(13, "foo.F90")).ptype - 'mytype' - >>> ParseSource("myname", "mytype", ParseContext(13, "foo.F90")).name - 'myname' - >>> print("{}".format(ParseSource("myname", "mytype", ParseContext(13, "foo.F90")).context)) - foo.F90:14 - """ - - def __init__(self, name_in, type_in, context_in): - """Initialize this ParseSource object.""" - self.__name = name_in - self.__type = type_in - self.__context = context_in - - @property - def ptype(self): - """Return this source's type""" - return self.__type - - @property - def name(self): - """Return this source's name""" - return self.__name - - @property - def context(self): - """Return this source's context""" - return self.__context - -######################################################################## diff --git a/scripts/parse_tools/preprocess.py b/scripts/parse_tools/preprocess.py deleted file mode 100755 index 06b94147..00000000 --- a/scripts/parse_tools/preprocess.py +++ /dev/null @@ -1,425 +0,0 @@ -#! /usr/bin/env python3 -""" -Classes to parse C preprocessor lines and to maintain a stack to allow -inclusion and exclusion of lines based on preprocessor symbol definitions. -""" - -# Python library imports -import re -import ast -# CCPP Framewor imports -from parse_source import ParseSyntaxError - -__defined_re__ = re.compile(r"defined\s+([A-Za-z0-9_]+)") - -############################################################################### - -class PreprocError(ValueError): - """Class to report preprocessor line errors""" - def __init__(self, message): - super(PreprocError, self).__init__(message) - -######################################################################## - -def preproc_bool(value): - """Turn a preprocessor value into a boolean""" - if isinstance(value, bool): - line_val = value - else: - try: - ival = int(value) - line_val = ival != 0 - except ValueError: - line_val = value != "0" - # end try - # end if - return line_val - -######################################################################## - -def preproc_item_value(item, preproc_defs): - """Find the value of a preproc (part of a parsed - preprocessor line)""" - value = False - if isinstance(item, ast.Expr): - value = preproc_item_value(item.value, preproc_defs) - elif isinstance(item, ast.Call): - func = item.func.id - # The only 'function' we know how to process is "defined" - if func == "defined": - args = item.args - if len(args) != 1: - raise PreprocError("Invalid defined statement, {}".format(ast.dump(item))) - # end if - symbol = args[0].id - # defined is True as long as we know about the symbol - value = symbol in preproc_defs - elif func == "notdefined": - args = item.args - if len(args) != 1: - raise PreprocError("Invalid defined statement, {}".format(ast.dump(item))) - # end if - symbol = args[0].id - # notdefined is True as long as we do not know about the symbol - value = symbol not in preproc_defs - else: - raise PreprocError("Cannot parse function {}".format(func)) - # end if - elif isinstance(item, ast.BoolOp): - left_val = preproc_item_value(item.values[0], preproc_defs) - right_val = preproc_item_value(item.values[1], preproc_defs) - oper = item.op - if isinstance(oper, ast.And): - value = preproc_bool(left_val) and preproc_bool(right_val) - elif isinstance(oper, ast.Or): - value = preproc_bool(left_val) or preproc_bool(right_val) - else: - raise PreprocError("Unknown binary operator, {}".format(oper)) - # end if - elif isinstance(item, ast.UnaryOp): - val = preproc_item_value(item.operand, preproc_defs) - oper = item.op - if isinstance(oper, ast.Not): - value = not preproc_bool(val) - else: - raise PreprocError("Unknown unary operator, {}".format(oper)) - # end if - elif isinstance(item, ast.Compare): - left_val = preproc_item_value(item.left, preproc_defs) - value = True - for index in range(len(item.ops)): - oper = item.ops[index] - rcomp = item.comparators[index] - right_val = preproc_item_value(rcomp, preproc_defs) - if isinstance(oper, ast.Eq): - value = value and (left_val == right_val) - elif isinstance(oper, ast.NotEq): - value = value and (left_val != right_val) - else: - # What remains are numerical comparisons, use integers - try: - ilval = int(left_val) - irval = int(right_val) - if isinstance(oper, ast.Gt): - value = value and (ilval > irval) - elif isinstance(oper, ast.GtE): - value = value and (ilval >= irval) - elif isinstance(oper, ast.Lt): - value = value and (ilval < irval) - elif isinstance(oper, ast.LtE): - value = value and (ilval <= irval) - else: - emsg = "Unknown comparison operator, {}" - raise PreprocError(emsg.format(oper)) - # end if - except ValueError: - value = False - # end try - # end if - # end for - elif isinstance(item, ast.Name): - id_key = item.id - if id_key in preproc_defs: - value = preproc_defs[id_key] - else: - value = id_key - # end if - elif isinstance(item, ast.Num): - value = item.n - else: - raise PreprocError("Cannot parse {}".format(item)) - # end if - return value - -######################################################################## - -def parse_preproc_line(line, preproc_defs): - """Parse a preprocessor line into a tree that can be evaluated""" - # Scan line and translate to python syntax - inchar = None # Character context - line_len = len(line) - pline = "" - index = 0 - while index < line_len: - if (line[index] == '"') or (line[index] == "'"): - if inchar == line[index]: - inchar = None - elif inchar is None: - inchar = line[index] - # Else in character context, just copy - # end if - pline = pline + line[index] - elif inchar is not None: - # In character context, just copy current character - pline = pline + line[index] - elif line[index:index+2] == '&&': - pline = pline + 'and' - index = index + 1 - elif line[index:index+2] == '||': - pline = pline + 'or' - index = index + 1 - elif line[index] == '!': - pline = pline + "not" - else: - match = __defined_re__.match(line[index:]) - if match is None: - # Just copy current character - pline = pline + line[index] - else: - mlen = len(match.group(0)) - pline = pline + "defined ({})".format(match.group(1)) - index = index + mlen - 1 - # end if - # end if - index = index + 1 - # end while - try: - ast_line = ast.parse(pline) - # We should only have one 'statement' - if len(ast_line.body) != 1: - line_val = False - success = False - else: - value = preproc_item_value(ast_line.body[0], preproc_defs) - line_val = preproc_bool(value) - success = True - # end if - except SyntaxError: - line_val = False - success = False - # end try - return line_val, success - -######################################################################## - -class PreprocStack(object): - """Class to handle preprocess regions""" - - ifdef_re = re.compile(r"#\s*ifdef\s+(.*)") - ifndef_re = re.compile(r"#\s*ifndef\s+(.*)") - if_re = re.compile(r"#\s*if([^dn].*)") - elif_re = re.compile(r"#\s*elif\s(.*)") - ifelif_re = re.compile(r"#\s*(?:el)?if\s(.*)") - else_re = re.compile(r"#\s*else") - end_re = re.compile(r"#\s*endif") - define_re = re.compile(r"#\s*define\s+([A-Za-z0-9_]+)\s+([^\s]*)") - undef_re = re.compile(r"#\s*undef\s+([A-Za-z0-9_]+)") - - def __init__(self): - """Initialize our region stack""" - self._region_stack = list() - - @staticmethod - def process_if_line(line, preproc_defs): - """Decide if (el)?if represents a True or False condition. - Return True iff the line evaluates to a True condition. - is a dictionary where each key is a symbol which - can be tested (e.g., 'FOO' in #ifdef FOO). The value is that - symbol's preprocessor value, if provided (e.g., 3 for -DFOO=3), - otherwise, it is None. - Return second logical value of False if we are unable to process - >>> PreprocStack().process_if_line("#if 0", {'CCPP':1}) - (False, True) - >>> PreprocStack().process_if_line("#if 1", {'CCPP':1}) - (True, True) - >>> PreprocStack().process_if_line("#elif 0", {'CCPP':1}) - (False, True) - >>> PreprocStack().process_if_line("#elif 1", {'CCPP':1}) - (True, True) - >>> PreprocStack().process_if_line("#if ( WRF_CHEM == 1 )", {'CCPP':1}) - (False, True) - >>> PreprocStack().process_if_line("#if ( WRF_CHEM == 1 )", {'WRF_CHEM':1}) - (True, True) - >>> PreprocStack().process_if_line("#if ( WRF_CHEM == 1 )", {'WRF_CHEM':0}) - (False, True) - >>> PreprocStack().process_if_line("#if (WRF_CHEM == 0)", {'CCPP':1}) - (False, True) - >>> PreprocStack().process_if_line("#if (WRF_CHEM == 0)", {'WRF_CHEM':1}) - (False, True) - >>> PreprocStack().process_if_line("#if (WRF_CHEM == 0)", {'WRF_CHEM':0}) - (True, True) - >>> PreprocStack().process_if_line("#if defined(CCPP)", {'CCPP':1}) - (True, True) - >>> PreprocStack().process_if_line("#if defined(CCPP)", {'CCPP':0}) - (True, True) - >>> PreprocStack().process_if_line("#if defined(CCPP)", {}) - (False, True) - >>> PreprocStack().process_if_line("#if ( defined WACCM_PHYS )", {'CCPP':1}) - (False, True) - >>> PreprocStack().process_if_line("#if (defined WACCM_PHYS)", {'WACCM_PHYS':1}) - (True, True) - >>> PreprocStack().process_if_line("#if (defined WACCM_PHYS)", {'WACCM_PHYS':0}) - (True, True) - >>> PreprocStack().process_if_line("#if (defined(DM_PARALLEL) && (! defined(STUBMPI)))", {}) - (False, True) - >>> PreprocStack().process_if_line("#if (defined(DM_PARALLEL) && (! defined(STUBMPI)))", {'DM_PARALLEL':1}) - (True, True) - >>> PreprocStack().process_if_line("#if (defined(DM_PARALLEL) && (! defined(STUBMPI)))", {'DM_PARALLEL':1, 'STUBMPI':0}) - (False, True) - >>> PreprocStack().process_if_line("#if (defined(DM_PARALLEL) && (! defined(STUBMPI)))", {'STUBMPI':0}) - (False, True) - >>> PreprocStack().process_if_line("# if (defined(DM_PARALLEL) || (! defined(STUBMPI)))", {}) - (True, True) - >>> PreprocStack().process_if_line("#if (defined(DM_PARALLEL) || (! defined(STUBMPI)))", {'DM_PARALLEL':1}) - (True, True) - >>> PreprocStack().process_if_line("#if (defined(DM_PARALLEL) || (! defined(STUBMPI)))", {'DM_PARALLEL':1, 'STUBMPI':0}) - (True, True) - >>> PreprocStack().process_if_line("#if (defined(DM_PARALLEL) || (! defined(STUBMPI)))", {'STUBMPI':0}) - (False, True) - >>> PreprocStack().process_if_line("#elif ( WRF_CHEM == 1 )", {'CCPP':1}) - (False, True) - >>> PreprocStack().process_if_line("#elif ( WRF_CHEM == 1 )", {'WRF_CHEM':1}) - (True, True) - >>> PreprocStack().process_if_line("#elif ( WRF_CHEM == 1 )", {'WRF_CHEM':0}) - (False, True) - >>> PreprocStack().process_if_line("#elif (WRF_CHEM == 0)", {'CCPP':1}) - (False, True) - >>> PreprocStack().process_if_line("# elif (WRF_CHEM == 0)", {'WRF_CHEM':1}) - (False, True) - >>> PreprocStack().process_if_line("#elif (WRF_CHEM == 0)", {'WRF_CHEM':0}) - (True, True) - >>> PreprocStack().process_if_line("#if defined(CCPP) &&", {'CCPP':1}) - (False, False) - """ - match = PreprocStack.ifelif_re.match(line) - if match is None: - return False, False # This is not a preproc line - # end if - value, okay = parse_preproc_line(match.group(1).strip(), preproc_defs) - return value, okay - - def process_line(self, line, preproc_defs, pobj, logger): - """Read and return if it is a preprocessor line. - In addition, if it is a preprocessor line enter an appropriate region - if indicated by .""" - sline = line.strip() - is_preproc_line = PreprocStack.is_preproc_line(line) - if is_preproc_line and (preproc_defs is not None): - match = PreprocStack.ifdef_re.match(sline) - if match is not None: - start_region = match.group(1) in preproc_defs - if start_region and (logger is not None): - lmsg = "Preproc: Starting True region ({}) on line {}" - logger.debug(lmsg.format(match.group(1), pobj)) - # end if - self.enter_region(start_region) - # end if - if match is None: - match = PreprocStack.ifndef_re.match(sline) - if match is not None: - start_region = match.group(1) not in preproc_defs - if (not start_region) and (logger is not None): - lmsg = "Preproc: Starting False region ({}) on line {}" - logger.debug(lmsg.format(match.group(1), pobj)) - # end if - self.enter_region(start_region) - # end if - # end if - if match is None: - match = PreprocStack.if_re.match(sline) - if match is not None: - line_val, success = self.process_if_line(sline, - preproc_defs) - self.enter_region(line_val) - if (not success) and (logger is not None): - lmsg = "WARNING: Preprocessor #if statement not handled, at {}" - logger.warning(lmsg.format(pobj)) - # end if - # end if - # end if - if match is None: - match = PreprocStack.elif_re.match(sline) - if match is not None: - line_val, success = self.process_if_line(sline, - preproc_defs) - self.modify_region(line_val) - if (not success) and (logger is not None): - lmsg = "WARNING: Preprocessor #elif statement not handled, at {}" - logger.warning(lmsg.format(pobj)) - # end if - # end if - # end if - if match is None: - match = PreprocStack.else_re.match(sline) - if match is not None: - # Always try to use True for else, modify_region will set - # correct value - self.modify_region(True) - # end if - # end if - if match is None: - match = PreprocStack.end_re.match(sline) - if match is not None: - self.exit_region(pobj) - # end if - # end if - if (match is None) and self.in_true_region(): - match = PreprocStack.define_re.match(sline) - if match is not None: - # Add (or replace) a symbol to our defs - preproc_defs[match.group(1)] = match.group(2) - # end if - # end if - if (match is None) and self.in_true_region(): - match = PreprocStack.undef_re.match(sline) - if (match is not None) and (match.group(1) in preproc_defs): - # Remove a symbol from our defs - del preproc_defs[match.group(1)] - # end if - # end if - # Ignore all other lines - # end if - return is_preproc_line - - def enter_region(self, valid): - """Enter a new region (if, ifdef, ifndef) which may - currently be valid""" - self._region_stack.append([valid, valid]) - - def exit_region(self, pobj): - """Leave the current (innermost) region""" - if not self._region_stack: - emsg = "#endif found with no matching #if, #ifdef, or #ifndef" - raise ParseSyntaxError(emsg, context=pobj) - # end if - self._region_stack.pop() - - def modify_region(self, valid): - """Possibly modify the current (innermost) region. - A region can be modified from False to True. - Any attempt to modify a region which has been True results in False - because after a region has been True, any #elif or #else must skipped. - """ - curr_region = self._region_stack.pop() - if curr_region[0]: - self._region_stack.append([curr_region[0], False]) - else: - self._region_stack.append([curr_region[0], valid]) - # end if - - def in_true_region(self): - """Return True iff the current line should be processed""" - true_region = True - for region in self._region_stack: - if not region[1]: - true_region = False - break - # end if - # end for - return true_region - - @staticmethod - def is_preproc_line(line): - """Return True iff line appears to be a preprocessor line""" - return line.lstrip()[0] == '#' - -######################################################################## - -if __name__ == "__main__": - # pylint: disable=ungrouped-imports - import doctest - import sys - # pylint: enable=ungrouped-imports - fail, _ = doctest.testmod() - sys.exit(fail) -# end if diff --git a/scripts/parse_tools/xml_tools.py b/scripts/parse_tools/xml_tools.py deleted file mode 100644 index d9210038..00000000 --- a/scripts/parse_tools/xml_tools.py +++ /dev/null @@ -1,572 +0,0 @@ -#!/usr/bin/env python3 - -""" -Parse a host-model registry XML file and return the captured variables. -""" - -# Python library imports -import os -import re -import shutil -import subprocess -import sys -import xml.etree.ElementTree as ET -import xml.dom.minidom -sys.path.insert(0, os.path.dirname(__file__)) -# CCPP framework imports -from parse_source import CCPPError -from parse_log import init_log, set_log_to_null - -# Global data -_INDENT_STR = " " -beg_tag_re = re.compile(r"([<][^/][^<>]*[^/][>])") -end_tag_re = re.compile(r"([<][/][^<>/]+[>])") -simple_tag_re = re.compile(r"([<][^/][^<>/]+[/][>])") - -# Find python version -PYSUBVER = sys.version_info[1] -_LOGGER = None - -############################################################################### -class XMLToolsInternalError(ValueError): -############################################################################### - """Error class for reporting internal errors""" - def __init__(self, message): - """Initialize this exception""" - super().__init__(message) - -############################################################################### -def find_schema_version(root): -############################################################################### - """ - Find the version of the host registry file represented by root - >>> find_schema_version(ET.fromstring('')) - [1, 0] - >>> find_schema_version(ET.fromstring('')) - [2, 0] - >>> find_schema_version(ET.fromstring('')) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: Illegal version string, '1.a' - Format must be . - >>> find_schema_version(ET.fromstring('')) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: Illegal version string, '0.0' - Major version must be at least 1 - >>> find_schema_version(ET.fromstring('')) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: Illegal version string, '0.0' - Minor version must be at least 0 - """ - verbits = None - if 'version' not in root.attrib: - raise CCPPError("version attribute required") - # end if - version = root.attrib['version'] - versplit = version.split('.') - try: - if len(versplit) != 2: - raise CCPPError('oops') - # end if (no else needed) - try: - verbits = [int(x) for x in versplit] - except ValueError as verr: - raise CCPPError(verr) from verr - # end try - if verbits[0] < 1: - raise CCPPError('Major version must be at least 1') - # end if - if verbits[1] < 0: - raise CCPPError('Minor version must be non-negative') - # end if - except CCPPError as verr: - errstr = """Illegal version string, '{}' - Format must be .""" - ve_str = str(verr) - if ve_str: - errstr = ve_str + '\n' + errstr - # end if - raise CCPPError(errstr.format(version)) from verr - # end try - return verbits - -############################################################################### -def find_schema_file(schema_root, version, schema_path=None): -############################################################################### - """Find and return the schema file based on and - or return None. - If is present, use that as the directory to find the - appropriate schema file. Otherwise, just look in the current directory.""" - - verstring = '_'.join([str(x) for x in version]) - schema_filename = "{}_v{}.xsd".format(schema_root, verstring) - if schema_path: - schema_file = os.path.join(schema_path, schema_filename) - else: - schema_file = schema_filename - # end if - if os.path.exists(schema_file): - return schema_file - # end if - return None - -############################################################################### -def validate_xml_file(filename, schema_root, version, logger, schema_path=None): -############################################################################### - """ - Find the appropriate schema and validate the XML file, , - against it using xmllint - """ - # Check the filename - if not os.path.isfile(filename): - raise CCPPError("validate_xml_file: Filename, '{}', does not exist".format(filename)) - # end if - if not os.access(filename, os.R_OK): - raise CCPPError("validate_xml_file: Cannot open '{}'".format(filename)) - # end if - if os.path.isfile(schema_root): - # We already have a file, just use it - schema_file = schema_root - else: - if not schema_path: - # Find the schema, based on the model version - thispath = os.path.abspath(__file__) - pdir = os.path.dirname(os.path.dirname(os.path.dirname(thispath))) - schema_path = os.path.join(pdir, 'schema') - # end if - schema_file = find_schema_file(schema_root, version, schema_path) - if not (schema_file and os.path.isfile(schema_file)): - verstring = '.'.join([str(x) for x in version]) - emsg = f"""validate_xml_file: Cannot find schema for version {verstring}, - {schema_file} does not exist""" - raise CCPPError(emsg) - # end if - # end if - if not os.access(schema_file, os.R_OK): - emsg = "validate_xml_file: Cannot open schema, '{}'" - raise CCPPError(emsg.format(schema_file)) - # end if - - # Find xmllint - xmllint = shutil.which('xmllint') # Blank if not installed - if not xmllint: - msg = "xmllint not found, could not validate file {}" - raise CCPPError("validate_xml_file: " + msg.format(filename)) - # end if - - # Validate XML file against schema - logger.debug("Checking file {} against schema {}".format(filename, - schema_file)) - cmd = [xmllint, '--noout', '--schema', schema_file, filename] - cproc = subprocess.run(cmd, check=False, capture_output=True) - if cproc.returncode == 0: - # We got a pass return code but some versions of xmllint do not - # correctly return an error code on non-validation so double check - # the result - result = b'validates' in cproc.stdout or b'validates' in cproc.stderr - else: - result = False - # end if - if result: - logger.debug(cproc.stdout) - logger.debug(cproc.stderr) - return result - else: - cmd = ' '.join(cmd) - outstr = f"Execution of '{cmd}' failed with code: {cproc.returncode}\n" - if cproc.stdout: - outstr += f"{cproc.stdout.decode('utf-8', errors='replace').strip()}\n" - if cproc.stderr: - outstr += f"{cproc.stderr.decode('utf-8', errors='replace').strip()}\n" - raise CCPPError(outstr) - -############################################################################### -def read_xml_file(filename, logger=None): -############################################################################### - """Read the XML file, , and return its tree and root - - Parameters: - filename (str): The path to an XML file to read and search. - logger (logging.Logger, optional): Logger for warnings/errors. - - Returns: - tree (xml.etree.ElementTree): The element tree from the input file. - root (xml.etree.ElementTree.Element): The root element of tree. - - Raises: - CCPPError: If the file cannot be found or read. - """ - if os.path.isfile(filename) and os.access(filename, os.R_OK): - file_open = (lambda x: open(x, 'r', encoding='utf-8')) - with file_open(filename) as file_: - try: - tree = ET.parse(file_) - root = tree.getroot() - except ET.ParseError as perr: - emsg = "read_xml_file: Cannot read {}, {}" - raise CCPPError(emsg.format(filename, perr)) from perr - elif not os.access(filename, os.R_OK): - raise CCPPError("read_xml_file: Cannot open '{}'".format(filename)) - else: - emsg = "read_xml_file: Filename, '{}', does not exist" - raise CCPPError(emsg.format(filename)) - # end if - if logger: - logger.debug(f"Reading XML file {filename}") - # end if - return tree, root - -############################################################################### -def load_suite_by_name(suite_name, group_name, file, logger=None): -############################################################################### - """ - Load a suite by its name, or a group of a suite by the suite and group names. - - Parameters: - suite_name (str): The name of the suite to find. - group_name (str or None): The name of the group to find within the suite. - file (str): The path to an XML file to read and search. - logger (logging.Logger, optional): Logger for warnings/errors. - - Returns: - xml.etree.ElementTree.Element: The matching suite or group element. - - Raises: - CCPPError: If the suite or group is not found, or if the schema is invalid. - - Examples: - >>> import tempfile - >>> import xml.etree.ElementTree as ET - >>> logger = init_log('xml_tools') - >>> set_log_to_null(logger) - >>> # Create temporary files for the nested suites - >>> tmpdir = tempfile.TemporaryDirectory() - >>> file1_path = os.path.join(tmpdir.name, "file1.xml") - >>> # Write XML contents to temporary file - >>> with open(file1_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... - ... - ... ''') - >>> load_suite_by_name("physics_suite", None, file1_path, logger).tag - 'suite' - >>> load_suite_by_name("physics_suite", "dynamics", file1_path, logger).attrib['name'] - 'dynamics' - >>> load_suite_by_name("physics_suite", "missing_group", file1_path, logger) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - CCPPError: Nested suite physics_suite, group missing_group, not found - >>> load_suite_by_name("missing_suite", None, file1_path, logger) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - CCPPError: Nested suite missing_suite not found - >>> tmpdir.cleanup() - """ - _, root = read_xml_file(file, logger) - schema_version = find_schema_version(root) - res = validate_xml_file(file, 'suite', schema_version, logger) - if not res: - raise CCPPError(f"Invalid suite definition file, '{file}'") - suite = root - if suite.attrib.get("name") == suite_name: - if group_name: - for group in suite.findall("group"): - if group.attrib.get("name") == group_name: - return group - else: - return suite - emsg = f"Nested suite {suite_name}" \ - + (f", group {group_name}," if group_name else "") \ - + " not found" + (f" in file {file}" if file else "") - raise CCPPError(emsg) - -############################################################################### -def replace_nested_suite(element, nested_suite, default_path, logger): -############################################################################### - """ - Replace a tag with the actual suite or group it references. - - This function looks up a referenced suite or suite group from an external - file, deep copies its children, and replaces the element - in the parent `element` with the copied contents. - - Parameters: - element (xml.etree.ElementTree.Element): The parent element containing the nested suite. - nested_suite (xml.etree.ElementTree.Element): The element to be replaced. - default_path (str): The default path to look for nested SDFs if file is not a absolute path. - logger (logging.Logger or None): Logger to record debug information. - - Returns: - str: The name of the suite that was replaced - - Example: - >>> import tempfile - >>> import xml.etree.ElementTree as ET - >>> from types import SimpleNamespace - >>> logger = init_log('xml_tools') - >>> set_log_to_null(logger) - >>> tmpdir = tempfile.TemporaryDirectory() - >>> file1_path = os.path.join(tmpdir.name, "file1.xml") - >>> with open(file1_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... my_scheme - ... - ... - ... ''') - >>> # Import nested suite at suite level - >>> xml = f''' - ... - ... - ... - ... ''' - >>> top_suite = ET.fromstring(xml) - >>> nested = top_suite.find("nested_suite") - >>> replace_nested_suite(top_suite, nested, tmpdir.name, logger) - 'my_suite' - >>> [child.tag for child in top_suite] - ['group'] - >>> top_suite.find("group").find("scheme").text - 'my_scheme' - >>> # Import group from nested suite at group level - >>> xml = f''' - ... - ... - ... - ... - ... - ... ''' - >>> top_suite = ET.fromstring(xml) - >>> top_group = top_suite.find("group") - >>> nested = top_group.find("nested_suite") - >>> replace_nested_suite(top_group, nested, tmpdir.name, logger) - 'my_suite' - >>> [child.tag for child in top_suite] - ['group'] - >>> top_suite.find("group").find("scheme").text - 'my_scheme' - >>> # Import group from nested suite at suite level - >>> xml = f''' - ... - ... - ... - ... ''' - >>> top_suite = ET.fromstring(xml) - >>> nested = top_suite.find("nested_suite") - >>> replace_nested_suite(top_suite, nested, tmpdir.name, logger) - 'my_suite' - >>> [child.tag for child in top_suite] - ['group'] - >>> top_suite.find("group").find("scheme").text - 'my_scheme' - >>> tmpdir.cleanup() - """ - suite_name = nested_suite.attrib.get("name") - group_name = nested_suite.attrib.get("group") - file = nested_suite.attrib.get("file") - if not os.path.isabs(file): - file = os.path.join(default_path, file) - referenced_suite = load_suite_by_name(suite_name, group_name, file, - logger=logger) - imported_content = [ET.fromstring(ET.tostring(child)) - for child in referenced_suite] - # Swap nested suite with imported content - for item in imported_content: - # If we are inserting a nested suite at the suite level (element.tag is suite), - # but we only want one group (group_name is not none), then we need to wrap - # the item in a group element. If on the other hand we insert an entire suite - # (all groups) at the suite level, or a specific group at the group level, - # then we can insert the item as is. - if element.tag == 'suite' and group_name: - item_to_insert = ET.Element("group", attrib={"name": group_name}) - item_to_insert.append(item) - else: - item_to_insert = item - element.insert(list(element).index(nested_suite), item_to_insert) - element.remove(nested_suite) - if logger: - msg = f"Expanded nested suite '{suite_name}'" \ - + (f", group '{group_name}'," if group_name else "") \ - + (f" in file '{file}'" if file else "") - logger.debug(msg.rstrip(',')) - # Return the name of the suite that we just replaced - return suite_name - -############################################################################### -def expand_nested_suites(suite, default_path, logger=None): -############################################################################### - """ - Recursively expand all elements within the XML element. - - This function finds elements within or elements, - and replaces them with the corresponding content from another suite. - - This operation is recursive and will continue expanding until no - elements remain. - - Parameters: - suite (xml.etree.ElementTree.Element): The root element. - logger (logging.Logger, optional): Logger for debug messages. - - Returns: - None. The XML tree is modified in place. - - Example: - >>> import tempfile - >>> import xml.etree.ElementTree as ET - >>> logger = init_log('xml_tools') - >>> set_log_to_null(logger) - >>> tmpdir = tempfile.TemporaryDirectory() - >>> file1_path = os.path.join(tmpdir.name, "file1.xml") - >>> file2_path = os.path.join(tmpdir.name, "file2.xml") - >>> file3_path = os.path.join(tmpdir.name, "file3.xml") - >>> file4_path = os.path.join(tmpdir.name, "file4.xml") - >>> file5_path = os.path.join(tmpdir.name, "file5.xml") - >>> # Write mock XML contents for the nested suites - >>> with open(file1_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... cloud_scheme - ... - ... - ... ''') - >>> with open(file2_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... pbl_scheme - ... - ... - ... ''') - >>> with open(file3_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... rrtmg_lw_scheme - ... - ... - ... rrtmg_sw_scheme - ... - ... - ... ''') - >>> with open(file4_path, "w") as f: - ... _ = f.write(f''' - ... - ... - ... - ... ''') - >>> with open(file5_path, "w") as f: - ... _ = f.write(f''' - ... - ... - ... - ... ''') - >>> # Parent suite - >>> xml_content = f''' - ... - ... - ... - ... - ... - ... - ... - ... ''' - >>> suite = ET.fromstring(xml_content) - >>> expand_nested_suites(suite, tmpdir.name, logger) - >>> ET.dump(suite) - - - cloud_scheme - - pbl_scheme - - rrtmg_lw_scheme - - rrtmg_sw_scheme - - >>> # Test infite recursion - >>> xml_content = f''' - ... - ... - ... - ... - ... - ... - ... ''' - >>> suite = ET.fromstring(xml_content) - >>> expand_nested_suites(suite, tmpdir.name, logger) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - CCPPError: Exceeded number of iterations while expanding nested suites - >>> tmpdir.cleanup() - """ - # To avoid infinite recursion, we simply count the number - # of iterations and stop at a certain limit. If someone is - # smart enough to come up with nested suite constructs that - # require more iterations, than he/she should be able to - # track down this variable and adjust it! - max_iterations = 10 - # Collect the names of the expanded suites - suite_names = [] - # Iteratively expand nested suites until they are all gone - keep_expanding = True - for num_iterations in range(max_iterations): - keep_expanding = False - # First, search all groups for nested_suite elements - groups = suite.findall("group") - for group in groups: - nested_suites = group.findall("nested_suite") - for nested in nested_suites: - suite_names.append(replace_nested_suite(group, nested, default_path, logger)) - # Trigger another pass over the root element - keep_expanding = True - # Second, search all suites for nested_suite elements - nested_suites = suite.findall("nested_suite") - for nested in nested_suites: - suite_names.append(replace_nested_suite(suite, nested, default_path, logger)) - # Trigger another pass over the root element - keep_expanding = True - if not keep_expanding: - return - raise CCPPError("Exceeded number of iterations while expanding nested suites. " + \ - "Check for infinite recursion or adjust limit max_iterations. " + \ - f"Suites expanded so far: {suite_names}") - -############################################################################### -def write_xml_file(root, file_path, logger=None): -############################################################################### - """Pretty-prints element root to an ASCII file using xml.dom.minidom""" - - def remove_whitespace_nodes(node): - """Helper function to recursively remove all text nodes that contain - only whitespace, which eliminates blank lines in the output.""" - for child in list(node.childNodes): - if child.nodeType == child.TEXT_NODE and not child.data.strip(): - node.removeChild(child) - elif child.hasChildNodes(): - remove_whitespace_nodes(child) - - # Convert ElementTree to a byte string - byte_string = ET.tostring(root, 'us-ascii') - - # Parse string using minidom for pretty printing - reparsed = xml.dom.minidom.parseString(byte_string) - - # Clean whitespace-only text nodes - remove_whitespace_nodes(reparsed) - - # Generate pretty-printed XML string - pretty_xml = reparsed.toprettyxml(indent=" ") - - # Write to file - with open(file_path, 'w', errors='xmlcharrefreplace') as f: - f.write(pretty_xml) - - # Tell everyone! - if logger: - logger.debug(f"Writing XML file {file_path}") - -############################################################################## diff --git a/scripts/state_machine.py b/scripts/state_machine.py deleted file mode 100755 index 9c26afd1..00000000 --- a/scripts/state_machine.py +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env python3 -# - -"""Classes and methods to implement a simple state machine.""" - -# Python library imports -import re -from collections import OrderedDict -# CCPP framework imports -from parse_tools import FORTRAN_ID - -############################################################################### - -class StateMachine: - """Class and methods to implement a simple state machine. - Note, a collections.UserDict would be nice here but it is not in python 2. - >>> StateMachine() - StateMachine() - >>> StateMachine([('ab','a','b','a')]) - StateMachine(ab) - >>> StateMachine([('ab','a','b','a'),('cd','c','d','c')]) - StateMachine(ab, cd) - >>> StateMachine([('ab','a','b','a')]).add_transition('cd','c','d','c') - - >>> StateMachine([('ab','a','b','a')])['cd'] = ('c','d','c') - >>> StateMachine([('ab','a','b','a'),('cd','c','d','c')]).transitions() - ['ab', 'cd'] - >>> StateMachine([('ab','a','b','a')]).initial_state('ab') - 'a' - >>> StateMachine([('ab','a','b','a')]).final_state('ab') - 'b' - >>> StateMachine([('ab','a','b','a')]).transition_regex('ab') - re.compile('a$', re.IGNORECASE) - >>> StateMachine([('ab','a','b','a')]).function_match('foo_a', transition='ab') - ('foo', 'a', 'ab') - >>> StateMachine([('ab','a','b',r'ax?')]).function_match('foo_a', transition='ab') - ('foo', 'a', 'ab') - >>> StateMachine([('ab','a','b',r'ax?')]).function_match('foo_ax', transition='ab') - ('foo', 'ax', 'ab') - >>> StateMachine([('ab','a','b','a')]).function_match('foo_ab', transition='ab') - (None, None, None) - >>> StateMachine([('ab','a','b','a'),('cd','c','d','c')]).function_match('foo_c') - ('foo', 'c', 'cd') - >>> StateMachine([('ab','a','b',r'ax?')]).transition_match('a') - 'ab' - >>> StateMachine([('ab','a','b',r'ax?')]).transition_match('ax') - 'ab' - >>> StateMachine([('ab','a','b',r'ax?')]).transition_match('axx') - - >>> StateMachine([('ab','a','b','a')]).transition_match('ab') - - >>> StateMachine([('ab','a','b','a'),('cd','c','d','c')]).transition_match('c') - 'cd' - >>> StateMachine((('ab','a','b','a'),)).add_transition('ab','c','d','c') #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ValueError: ERROR: transition, 'ab', already exists - >>> StateMachine((('ab','a','b','a'))) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ValueError: Invalid initial_data transition ('ab'), should be of the form (name, inital_state, final_state, regex). - >>> StateMachine([('ab','a','b','a')])['cd'] = ('c','d') #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ValueError: Invalid transition (('c', 'd')), should be of the form (inital_state, final_state, regex). - """ - - def __init__(self, initial_data=None): - """Implement a finite state machine. - is an iterable where each item has four elements: - (transition_name, , , ) - is a string representing allowable names for - functions which form part of the transition action. - """ - # Implement the State Transition Table as a tuple and use accessors - self.__stt__ = OrderedDict() - if initial_data is not None: - # Note that we need to add states with longer regular expressions - # before short ones so that we match correctly. - for trans in sorted(initial_data, key=lambda x: len(x[3]) if len(x) > 3 else 0, reverse=True): - if len(trans) != 4: - raise ValueError("Invalid initial_data transition ({}), should be of the form (name, inital_state, final_state, regex).".format(trans)) - # end if - self.add_transition(trans[0], trans[1], trans[2], trans[3]) - # end for - # end if - - def add_transition(self, name, init_state, final_state, regex): - """Add a transition to this state machine. - See __setitem__ for implementation details.""" - self[name] = (init_state, final_state, regex) - - def transitions(self): - """Return a list of transition names""" - return list(self.__stt__.keys()) - - def initial_state(self, transition): - """Return the initial (before) state for """ - return self.__stt__[transition][0] - - def final_state(self, transition): - """Return the final (after) state for """ - return self.__stt__[transition][1] - - def transition_regex(self, transition): - """Return the compiled regex for """ - return self.__stt__[transition][2] - - def function_regex(self, transition): - """Return the compiled functino regex for """ - return self.__stt__[transition][3] - - def transition_match(self, test_str, transition=None): - """Return the matched transition, if found. - """ - match_trans = None - if transition is None: - trans_list = self.transitions() - else: - trans_list = [transition] - # end if - for trans in trans_list: - regex = self.transition_regex(trans) - match = regex.match(test_str) - if match is not None: - match_trans = trans - break - # end if - # end for - return match_trans - - def function_match(self, test_str, transition=None): - """Return a function ID, transition identifier, and matched - transition if found. - If is None, look for a match in any transition, - otherwise, only look for a specific match to that transition. - """ - if transition is None: - trans_list = self.transitions() - else: - trans_list = [transition] - # end if - func_id = None - trans_id = None - match_trans = None - for trans in trans_list: - regex = self.function_regex(trans) - match = regex.match(test_str) - if match is not None: - func_id = match.group(1) - trans_id = match.group(2) - match_trans = trans - break - # end if - # end for - return func_id, trans_id, match_trans - - def __getitem__(self, key): - return self.__stt__[key] - - def __setitem__(self, key, value): - if key in self.__stt__: - raise ValueError("ERROR: transition, '{}', already exists".format(key)) - # end if - if len(value) != 3: - raise ValueError("Invalid transition ({}), should be of the form (inital_state, final_state, regex).".format(value)) - # end if - regex = re.compile(value[2] + r"$", flags=re.IGNORECASE) - function = re.compile(FORTRAN_ID + r"_(" + value[2] + r")$", flags=re.IGNORECASE) - self.__stt__[key] = (value[0], value[1], regex, function) - - def __delitem__(self, key): - del self.__stt__[key] - - def __iter__(self): - return iter(self.__stt__) - - def __len__(self): - return len(self.__stt__) - - def __str__(self): - return "StateMachine({})".format(", ".join(self.transitions())) - - def __repr__(self): - return str(self) - -############################################################################### -if __name__ == "__main__": - # pylint: disable=ungrouped-imports - import doctest - import sys - # pylint: enable=ungrouped-imports - fail, _ = doctest.testmod() - sys.exit(fail) -# end if diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py deleted file mode 100755 index 3c846ed5..00000000 --- a/scripts/suite_objects.py +++ /dev/null @@ -1,2024 +0,0 @@ -#!/usr/bin/env python3 -# - -"""Classes and methods to create a Fortran suite-implementation file -to implement calls to a set of suites for a given host model.""" - -# Python library imports -import logging -import re -import xml.etree.ElementTree as ET -# CCPP framework imports -from ccpp_state_machine import CCPP_STATE_MACH, RUN_PHASE_NAME -from code_block import CodeBlock -from constituents import ConstituentVarDict -from framework_env import CCPPFrameworkEnv -from metavar import Var, VarDictionary, VarLoopSubst -from metavar import CCPP_CONSTANT_VARS, CCPP_LOOP_VAR_STDNAMES -from parse_tools import ParseContext, ParseSource, context_string -from parse_tools import ParseInternalError, CCPPError -from parse_tools import init_log, set_log_to_null -from var_props import is_horizontal_dimension, find_horizontal_dimension -from var_props import find_vertical_dimension -from var_props import VarCompatObj - -# pylint: disable=too-many-lines - -############################################################################### -# Module (global) variables -############################################################################### - -_OBJ_LOC_RE = re.compile(r"(0x[0-9A-Fa-f]+)>") -_BLANK_DIMS_RE = re.compile(r"[(][:](,:)*[)]$") - -# Source for internally generated variables. -_API_SOURCE_NAME = "CCPP_API" -# Use the constituent source type for consistency -_API_SUITE_VAR_NAME = ConstituentVarDict.constitutent_source_type() -_API_GROUP_VAR_NAME = "group" -_API_SCHEME_VAR_NAME = "scheme" -_API_LOCAL_VAR_NAME = "local" -_API_LOCAL_VAR_TYPES = [_API_LOCAL_VAR_NAME, _API_SUITE_VAR_NAME] -_API_CONTEXT = ParseContext(filename="ccpp_suite.py") -_API_SOURCE = ParseSource(_API_SOURCE_NAME, _API_SCHEME_VAR_NAME, _API_CONTEXT) -_API_LOCAL = ParseSource(_API_SOURCE_NAME, _API_LOCAL_VAR_NAME, _API_CONTEXT) -_API_LOGGING = init_log('ccpp_suite') -set_log_to_null(_API_LOGGING) -_API_DUMMY_RUN_ENV = CCPPFrameworkEnv(_API_LOGGING, - ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - -############################################################################### -def new_suite_object(item, context, parent, run_env, loop_count=0): -############################################################################### - "'Factory' method to create the appropriate suite object from XML" - new_item = None - if item.tag == 'subcycle': - new_item = Subcycle(item, context, parent, run_env, loop_count=loop_count) - elif item.tag == 'scheme': - new_item = Scheme(item, context, parent, run_env) - else: - emsg = "Unknown CCPP suite element type, '{}'" - raise CCPPError(emsg.format(item.tag)) - # end if - return new_item - -############################################################################### - -class CallList(VarDictionary): - """A simple class to hold a routine's call list (dummy arguments)""" - - def __init__(self, name, run_env, routine=None): - """Initialize this call list. - is the name of this dictionary. - is a pointer to the routine for which this is a call list - or None for a routine that is not a SuiteObject. - """ - self.__routine = routine - super().__init__(name, run_env) - - def add_vars(self, call_list, run_env, gen_unique=False): - """Add new variables from another CallList ()""" - for var in call_list.variable_list(): - stdname = var.get_prop_value('standard_name') - self.add_variable(var, run_env, gen_unique=gen_unique, adjust_intent=True, exists_ok=True) - # end for - - def add_variable(self, newvar, run_env, exists_ok=False, gen_unique=False, - adjust_intent=False): - """Add as for VarDictionary but make sure that the variable - has an intent with the default being intent(in). - """ - # We really need an intent on a dummy argument - if newvar.get_prop_value("intent") is None: - subst_dict = {'intent' : 'in'} - oldvar = newvar - newvar = oldvar.clone(subst_dict, source_name=self.name, - source_type=_API_GROUP_VAR_NAME, - context=oldvar.context) - # end if - super().add_variable(newvar, run_env, exists_ok=exists_ok, - gen_unique=gen_unique, adjust_intent=adjust_intent) - - def call_string(self, cldicts=None, is_func_call=False, subname=None, sub_lname_list=None): - """Return a dummy argument string for this call list. - may be a list of VarDictionary objects to search for - local_names (default is to use self). - should be set to True to construct a call statement. - If is False, construct a subroutine dummy argument - list. - may be a list of local_name substitutions. - """ - arg_str = "" - arg_sep = "" - for var in self.variable_list(): - # Do not include constants - stdname = var.get_prop_value('standard_name') - if stdname not in CCPP_CONSTANT_VARS: - # Find the dummy argument name - dummy = var.get_prop_value('local_name') - # Now, find the local variable name - if cldicts is not None: - for cldict in cldicts: - dvar = cldict.find_variable(standard_name=stdname, - any_scope=False) - if dvar is not None: - break - # end if - # end for - if dvar is None: - if subname is not None: - errmsg = "{}: ".format(subname) - else: - errmsg = "" - # end if - errmsg += "'{}', not found in call list for '{}'" - clnames = [x.name for x in cldicts] - raise CCPPError(errmsg.format(stdname, clnames)) - # end if - lname = dvar.get_prop_value('local_name') - # Optional variables in the caps are associated with - # local pointers of _ptr - if dvar.get_prop_value('optional'): - lname = dummy+'_ptr' - # end if - else: - cldict = None - aref = var.array_ref(local_name=dummy) - if aref is not None: - lname = aref.group(1) - else: - lname = dummy - # end if - # end if - # Modify Scheme call_list to handle local_name change for this var. - # Are there any variable transforms for this scheme? - # If so, change Var's local_name need to local dummy array containing - # transformed argument, var_trans_local. - if sub_lname_list: - for (var_trans_local, var_lname, sname, rindices, lindices, compat_obj) in sub_lname_list: - if (sname == stdname): - lname = var_trans_local - # end if - # end for - # end if - if is_func_call: - if cldicts is not None: - use_dicts = cldicts - else: - use_dicts = [self] - # end if - run_phase = self.routine.run_phase() - # We only need dimensions for suite variables in run phase - need_dims = SuiteObject.is_suite_variable(dvar) and run_phase - vdims = var.call_dimstring(var_dicts=use_dicts, - explicit_dims=need_dims, - loop_subst=run_phase) - if _BLANK_DIMS_RE.match(vdims) is None: - lname = lname + vdims - # end if - # end if - if is_func_call: - arg_str += "{}{}={}".format(arg_sep, dummy, lname) - else: - arg_str += "{}{}".format(arg_sep, lname) - # end if - arg_sep = ", " - # end if - # end for - return arg_str - - @property - def routine(self): - """Return the routine for this call list (or None)""" - return self.__routine - -############################################################################### - -class SuiteObject(VarDictionary): - """Base class for all CCPP Suite objects (e.g., Scheme, Subcycle) - SuiteObjects have an internal dictionary for variables created for - execution of the SuiteObject. These variables will be allocated and - managed at the Group level (unless cross-group usage or persistence - requires handling at the Suite level). - SuiteObjects also have a call list which is a list of variables which - are passed to callable SuiteObjects (e.g., Scheme). - """ - - def __init__(self, name, context, parent, run_env, - active_call_list=False, variables=None, phase_type=None): - # pylint: disable=too-many-arguments - self.__name = name - self.__context = context - self.__parent = parent - self.__run_env = run_env - if active_call_list: - self.__call_list = CallList(name + '_call_list', run_env, - routine=self) - else: - self.__call_list = None - # end if - self.__parts = list() - self.__needs_horizontal = None - self.__phase_type = phase_type - # Initialize our dictionary - super().__init__(self.name, run_env, - variables=variables, parent_dict=parent) - - def declarations(self): - """Return a list of local variables to be declared in parent Group - or Suite. By default, this list is the object's embedded VarDictionary. - """ - return self.variable_list() - - def add_part(self, item, replace=False): - """Add an object (e.g., Scheme, Subcycle) to this SuiteObject. - if is True, replace in its current position in self. - """ - if replace: - if item in self.__parts: - index = self.__parts.index(item) - else: - emsg = 'Cannot replace {} in {}, not a member' - raise ParseInternalError(emsg.format(item.name, self.name)) - # end if - else: - if item in self.__parts: - emsg = 'Cannot add {} to {}, already a member' - raise ParseInternalError(emsg.format(item.name, self.name)) - # end if - index = len(self.__parts) - # end if - # Just add - self.__parts.insert(index, item) - item.reset_parent(self) - - def schemes(self): - """Return a flattened list of schemes for this SuiteObject""" - schemes = list() - for item in self.__parts: - schemes.extend(item.schemes()) - # end for - return schemes - - def reset_parent(self, new_parent): - """Reset the parent of this SuiteObject (which has been moved)""" - self.__parent = new_parent - - def phase(self): - """Return the CCPP state phase_type for this SuiteObject""" - trans = self.phase_type - if trans is None: - if self.parent is not None: - trans = self.parent.phase() - else: - trans = False - # end if - # end if - return trans - - def run_phase(self): - """Return True iff this SuiteObject is in a run phase group""" - return self.phase() == RUN_PHASE_NAME - - def timestep_phase(self): - '''Return True iff this SuiteObject is in a timestep initial or - timestep final phase group''' - phase = self.phase() - return (phase is not None) and ('timestep' in phase) - - def register_action(self, vaction): - """Register (i.e., save information for processing during write stage) - and return True or pass up to the parent of - . Return True if any level registers , False otherwise. - The base class will not register any action, it must be registered in - an override of this method. - """ - if self.parent is not None: - return self.parent.register_action(vaction) - # end if - return False - - @classmethod - def is_suite_variable(cls, var): - """Return True iff belongs to our Suite""" - return var and (var.source.ptype == _API_SUITE_VAR_NAME) - - def is_local_variable(self, var): - """Return the local variable matching if one is found belonging - to this object or any of its SuiteObject parents.""" - stdname = var.get_prop_value('standard_name') - lvar = None - obj = self - while (not lvar) and (obj is not None) and isinstance(obj, SuiteObject): - lvar = obj.find_variable(standard_name=stdname, any_scope=False, - search_call_list=False) - if not lvar: - obj = obj.parent - # end if - # end while - return lvar - - def add_call_list_variable(self, newvar, exists_ok=False, - gen_unique=False, subst_dict=None): - """Add to this SuiteObject's call_list. If this SuiteObject - does not have a call list, recursively try the SuiteObject's parent - If is not None, create a clone using that as a dictionary - of substitutions. - Do not add if it exists as a local variable. - Do not add if it is a suite variable""" - stdname = newvar.get_prop_value('standard_name') - if self.parent: - pvar = self.parent.find_variable(standard_name=stdname, - source_var=newvar, - any_scope=False) - else: - pvar = None - # end if - if SuiteObject.is_suite_variable(pvar): - pass # Do not add suite variable to a call list - elif self.is_local_variable(newvar): - pass # Do not add to call list, it is owned by a SuiteObject - elif self.call_list is not None: - if (stdname in CCPP_LOOP_VAR_STDNAMES) and (not self.run_phase()): - errmsg = 'Attempting to use loop variable {} in {} phase' - raise CCPPError(errmsg.format(stdname, self.phase())) - # end if - # Do we need a clone? - if isinstance(self, Group): - stype = _API_GROUP_VAR_NAME - else: - stype = None - # end if - if stype or subst_dict: - oldvar = newvar - if subst_dict is None: - subst_dict = {} - # end if - # Make sure that this variable has an intent - if ((oldvar.get_prop_value("intent") is None) and - ("intent" not in subst_dict)): - subst_dict["intent"] = "in" - # end if - newvar = oldvar.clone(subst_dict, source_name=self.name, - source_type=stype, context=self.context) - # end if - self.call_list.add_variable(newvar, self.run_env, - exists_ok=exists_ok, - gen_unique=gen_unique, - adjust_intent=True) - # We need to make sure that this variable's dimensions are available - for vardim in newvar.get_dim_stdnames(include_constants=False): - # Unnamed dimensions are ok for allocatable variables - if vardim == '' and newvar.get_prop_value('allocatable'): - continue - elif vardim == '': - emsg = f"{self.name}: Cannot have unnamed/empty string dimension" - raise ParseInternalError(emsg) - # end if - dvar = self.find_variable(standard_name=vardim, - any_scope=True) - if dvar is None: - emsg = "{}: Could not find dimension {} in {}" - raise ParseInternalError(emsg.format(self.name, - vardim, stdname)) - # end if - elif self.parent is None: - errmsg = 'No call_list found for {}'.format(newvar) - raise ParseInternalError(errmsg) - elif pvar: - # Check for call list incompatibility - if pvar is not None: - compat, reason = pvar.compatible(newvar, self.run_env) - if not compat: - emsg = 'Attempt to add incompatible variable to call list:' - emsg += '\n{} from {} is not compatible with {} from {}' - nlreason = newvar.get_prop_value(reason) - plreason = pvar.get_prop_value(reason) - emsg += '\nreason = {} ({} != {})'.format(reason, - nlreason, - plreason) - nlname = newvar.get_prop_value('local_name') - plname = pvar.get_prop_value('local_name') - raise CCPPError(emsg.format(nlname, newvar.source.name, - plname, pvar.source.name)) - # end if - # end if (no else, variable already in call list) - else: - self.parent.add_call_list_variable(newvar, exists_ok=exists_ok, - gen_unique=gen_unique, - subst_dict=subst_dict) - # end if - - def add_variable_to_call_tree(self, var, vmatch=None, subst_dict=None): - """Add to 's call_list (or a parent if does not - have an active call_list). - If is not None, also add the loop substitution variables - which must be present. - If is not None, create a clone using that as a dictionary - of substitutions. - """ - found_dims = False - if var is not None: - self.add_call_list_variable(var, exists_ok=True, - gen_unique=True, subst_dict=subst_dict) - found_dims = True - # end if - if vmatch is not None: - svars = vmatch.has_subst(self, any_scope=True) - if svars is None: - found_dims = False - else: - found_dims = True - for svar in svars: - self.add_call_list_variable(svar, exists_ok=True) - # end for - # Register the action (probably at Group level) - self.register_action(vmatch) - # end if - # end if - return found_dims - - def horiz_dim_match(self, ndim, hdim, nloop_subst): - """Find a match between and , if they are both - horizontal dimensions. - If == , return . - If is not None and its required standard names exist - in our extended dictionary, return them. - Otherwise, return None. - NB: Loop substitutions are only allowed during the run phase but in - other phases, horizontal_dimension and horizontal_loop_extent - are the same. - """ - dim_match = None - nis_hdim = is_horizontal_dimension(ndim) - his_hdim = is_horizontal_dimension(hdim) - if nis_hdim and his_hdim: - if ndim == hdim: - dim_match = ndim - elif self.run_phase() and (nloop_subst is not None): - svars = nloop_subst.has_subst(self, any_scope=True) - match = svars is not None - if match: - if isinstance(self, Scheme): - obj = self.parent - else: - obj = self - # end if - for svar in svars: - obj.add_call_list_variable(svar, exists_ok=True) - # end for - dim_match = ':'.join(nloop_subst.required_stdnames) - # end if - elif not self.run_phase(): - if ((hdim == 'ccpp_constant_one:horizontal_dimension') and - (ndim == 'ccpp_constant_one:horizontal_loop_extent')): - dim_match = hdim - elif ((hdim == 'ccpp_constant_one:horizontal_dimension') and - (ndim == 'horizontal_loop_begin:horizontal_loop_end')): - dim_match = hdim - # end if (no else, there is no non-run-phase match) - # end if (no else, there is no match) - # end if (no else, there is no match) - return dim_match - - @staticmethod - def dim_match(need_dim, have_dim): - """Test whether matches . - If they match, return the matching dimension (which may be - modified by, e.g., a loop substitution). - If they do not match, return None. - """ - match = None - # First, try for all the marbles - if need_dim == have_dim: - match = need_dim - # end if - # Is one side missing a one start? - if not match: - ndims = need_dim.split(':') - hdims = have_dim.split(':') - if len(ndims) > len(hdims): - if ndims[0].lower == 'ccpp_constant_one': - ndims = ndims[1:] - elif hdims[0].lower == 'ccpp_constant_one': - hdims = hdims[1:] - # end if (no else) - # Last try - match = ndims == hdims - # end if - # end if - - return match - - def match_dimensions(self, need_dims, have_dims): - """Compare dimensions between and . - Return 6 items: - 1) Return True if all dims match. - If has a vertical dimension and does not - but all other dimensions match, return False but include the - missing dimension index as the third return value. - 2) Return modified, if necessary to - reflect the available limits. - 3) Return have_dims modified, if necessary to reflect - any loop substitutions. If no substitutions, return None - This is done so that the correct dimensions are used in the host cap. - 4) Return the name of the missing vertical index, or None - 5) Return a permutation array if the dimension ordering is - different (or None if the ordering is the same). Each element of the - permutation array is the index in for that dimension of - . - 6) Finally, return a 'reason' string. If match (first return value) is - False, this string will contain information about the reason for - the match failure. - >>> SuiteObject('foo', _API_CONTEXT, None, _API_DUMMY_RUN_ENV).match_dimensions(['horizontal_loop_extent'], ['horizontal_loop_extent']) - (True, ['horizontal_loop_extent'], ['horizontal_loop_extent'], None, '') - >>> SuiteObject('foo', _API_CONTEXT,None,_API_DUMMY_RUN_ENV,variables=[Var({'local_name':'beg','standard_name':'horizontal_loop_begin','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL,_API_DUMMY_RUN_ENV),Var({'local_name':'end','standard_name':'horizontal_loop_end','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL, _API_DUMMY_RUN_ENV)],active_call_list=True,phase_type='initialize').match_dimensions(['ccpp_constant_one:horizontal_loop_extent'], ['ccpp_constant_one:horizontal_dimension']) - (True, ['ccpp_constant_one:horizontal_dimension'], ['ccpp_constant_one:horizontal_dimension'], None, '') - >>> SuiteObject('foo', _API_CONTEXT,None,_API_DUMMY_RUN_ENV,variables=[Var({'local_name':'beg','standard_name':'horizontal_loop_begin','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL, _API_DUMMY_RUN_ENV),Var({'local_name':'end','standard_name':'horizontal_loop_end','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL, _API_DUMMY_RUN_ENV)],active_call_list=True,phase_type=RUN_PHASE_NAME).match_dimensions(['ccpp_constant_one:horizontal_loop_extent'], ['horizontal_loop_begin:horizontal_loop_end']) - (True, ['horizontal_loop_begin:horizontal_loop_end'], ['horizontal_loop_begin:horizontal_loop_end'], None, '') - >>> SuiteObject('foo', _API_CONTEXT,None,_API_DUMMY_RUN_ENV,variables=[Var({'local_name':'beg','standard_name':'horizontal_loop_begin','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL, _API_DUMMY_RUN_ENV),Var({'local_name':'end','standard_name':'horizontal_loop_end','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL, _API_DUMMY_RUN_ENV),Var({'local_name':'lev','standard_name':'vertical_layer_dimension','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL, _API_DUMMY_RUN_ENV)],active_call_list=True,phase_type=RUN_PHASE_NAME).match_dimensions(['ccpp_constant_one:horizontal_loop_extent','ccpp_constant_one:vertical_layer_dimension'], ['horizontal_loop_begin:horizontal_loop_end','ccpp_constant_one:vertical_layer_dimension']) - (True, ['horizontal_loop_begin:horizontal_loop_end', 'ccpp_constant_one:vertical_layer_dimension'], ['horizontal_loop_begin:horizontal_loop_end', 'ccpp_constant_one:vertical_layer_dimension'], None, '') - >>> SuiteObject('foo', _API_CONTEXT,None,_API_DUMMY_RUN_ENV,variables=[Var({'local_name':'beg','standard_name':'horizontal_loop_begin','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL, _API_DUMMY_RUN_ENV),Var({'local_name':'end','standard_name':'horizontal_loop_end','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL, _API_DUMMY_RUN_ENV),Var({'local_name':'lev','standard_name':'vertical_layer_dimension','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL, _API_DUMMY_RUN_ENV)],active_call_list=True,phase_type=RUN_PHASE_NAME).match_dimensions(['ccpp_constant_one:horizontal_loop_extent','ccpp_constant_one:vertical_layer_dimension'], ['ccpp_constant_one:vertical_layer_dimension','horizontal_loop_begin:horizontal_loop_end']) - (True, ['horizontal_loop_begin:horizontal_loop_end', 'ccpp_constant_one:vertical_layer_dimension'], ['ccpp_constant_one:vertical_layer_dimension', 'horizontal_loop_begin:horizontal_loop_end'], [1, 0], '') - """ - new_need_dims = [] - new_have_dims = list(have_dims) - perm = [] - match = True - reason = '' - nlen = len(need_dims) - hlen = len(have_dims) - _, nvdim_index = find_vertical_dimension(need_dims) - _, hvdim_index = find_vertical_dimension(have_dims) - _, nhdim_index = find_horizontal_dimension(need_dims) - _, hhdim_index = find_horizontal_dimension(have_dims) - if hhdim_index < 0 <= nhdim_index: - match = False - nlen = 0 # To skip logic below - hlen = 0 # To skip logic below - reason = '{hname}{hctx} is missing a horizontal dimension ' - reason += 'required by {nname}{nctx}' - # end if - for nindex in range(nlen): - neddim = need_dims[nindex] - if nindex == nhdim_index: - # Look for a horizontal dimension match - vmatch = VarDictionary.loop_var_match(neddim) - hmatch = self.horiz_dim_match(neddim, have_dims[hhdim_index], - vmatch) - if hmatch: - perm.append(hhdim_index) - new_need_dims.append(hmatch) - new_have_dims[hhdim_index] = hmatch - found_ndim = True - else: - found_ndim = False - # end if - else: - # Find the first dimension in have_dims that matches neddim - found_ndim = False - if nvdim_index < 0 <= hvdim_index: - skip = hvdim_index - else: - skip = -1 - # end if - hdim_indices = [x for x in range(hlen) - if (x not in perm) and (x != skip)] - for hindex in hdim_indices: - if (hindex != hvdim_index) or (nvdim_index >= 0): - hmatch = self.dim_match(neddim, have_dims[hindex]) - if hmatch: - perm.append(hindex) - new_need_dims.append(hmatch) - new_have_dims[hindex] = hmatch - found_ndim = True - break - # end if - # end if - # end if - # end for - if not found_ndim: - match = False - reason = 'Could not find dimension, ' + neddim + ', in ' - reason += '{hname}{hctx}. Needed by {nname}{nctx}' - break - # end if (no else, we are still okay) - # end for - perm_test = list(range(hlen)) - # If no permutation is found, reset to None - if perm == perm_test: - perm = None - elif (not match): - perm = None - # end if (else, return perm as is) - if new_have_dims == have_dims: - have_dims = None # Do not make any substitutions - # end if - return match, new_need_dims, new_have_dims, perm, reason - - def find_variable(self, standard_name=None, source_var=None, - any_scope=True, clone=None, - search_call_list=False, loop_subst=False): - """Find a matching variable to , create a local clone (if - is True), or return None. - First search the SuiteObject's internal dictionary, then its - call list (unless is True, then any parent - dictionary (if is True). - can be a Var object or a standard_name string. - is not used by this version of . - """ - # First, search our local dictionary - if standard_name is None: - if source_var is None: - emsg = "One of or must be passed." - raise ParseInternalError(emsg) - # end if - standard_name = source_var.get_prop_value('standard_name') - elif source_var is not None: - stest = source_var.get_prop_value('standard_name') - if stest != standard_name: - emsg = (" and must match if " + - "both are passed.") - raise ParseInternalError(emsg) - # end if - # end if - scl = search_call_list - stdname = standard_name - # Don't clone yet, might find the variable further down - found_var = super().find_variable(standard_name=stdname, - source_var=source_var, - any_scope=False, clone=None, - search_call_list=scl, - loop_subst=loop_subst) - if (not found_var) and (self.call_list is not None) and scl: - # Don't clone yet, might find the variable further down - found_var = self.call_list.find_variable(standard_name=stdname, - source_var=source_var, - any_scope=False, - clone=None, - search_call_list=scl, - loop_subst=loop_subst) - # end if - loop_okay = VarDictionary.loop_var_okay(stdname, self.run_phase()) - if not loop_okay: - loop_subst = False - # end if - if (found_var is None) and any_scope and (self.parent is not None): - # We do not have the variable, look to parents. - found_var = self.parent.find_variable(standard_name=stdname, - source_var=source_var, - any_scope=True, - clone=clone, - search_call_list=scl, - loop_subst=loop_subst) - # end if - return found_var - - def match_variable(self, var, run_env): - """Try to find a source for in this SuiteObject's dictionary - tree. Several items are returned: - found_var: True if a match was found - vert_dim: The vertical dimension in , or None - call_dims: How this variable should be called (or None if no match) - perm: Permutation (XXgoldyXX: Not yet implemented) - """ - vstdname = var.get_prop_value('standard_name') - vdims = var.get_dimensions() - if (not vdims) and self.run_phase(): - vmatch = VarDictionary.loop_var_match(vstdname) - else: - vmatch = None - # end if - - found_var = False - new_vdims = list() - var_vdim = var.has_vertical_dimension(dims=vdims) - compat_obj = None - dict_var = None - if var.get_prop_value('type') == 'ccpp_constituent_properties_t': - if self.phase() == 'register': - found_var = True - new_vdims = [':'] - return found_var, dict_var, var_vdim, new_vdims, compat_obj - else: - errmsg = "Variables of type ccpp_constituent_properties_t only allowed in register phase: " - sname = var.get_prop_value('standard_name') - errmsg += f"'{sname}' found in {self.phase()} phase" - raise CCPPError(errmsg) - # end if - # end if - - # Does this variable exist in the calling tree? - dict_var = self.find_variable(source_var=var, any_scope=True) - if dict_var is None: - # No existing variable but add loop var match to call tree - found_var = self.parent.add_variable_to_call_tree(dict_var, - vmatch=vmatch) - new_vdims = vdims - elif dict_var.source.ptype in _API_LOCAL_VAR_TYPES: - # We cannot change the dimensions of locally-declared variables - # Using a loop substitution is invalid because the loop variable - # value has not yet been set. - # Therefore, we have to use the declaration dimensions in the call. - found_var = True - new_vdims = dict_var.get_dimensions() - else: - # Check dimensions - dict_dims = dict_var.get_dimensions() - if vdims: - args = self.parent.match_dimensions(vdims, dict_dims) - match, new_vdims, new_dict_dims, perm, err = args - if perm is not None: - errmsg = "Permuted indices are not yet supported" - lname = var.get_prop_value('local_name') - dstr = ', '.join(vdims) - ctx = context_string(var.context) - errmsg += ", var = {}({}){}".format(lname, dstr, ctx) - raise CCPPError(errmsg) - # end if - else: - new_vdims = list() - new_dict_dims = dict_dims - match = True - # end if - # If variable is defined as "inactive" by the host, ensure that - # this variable is declared as "optional" by the scheme. If - # not satisfied, return error. - host_var_active = dict_var.get_prop_value('active') - scheme_var_optional = var.get_prop_value('optional') - if (not scheme_var_optional and host_var_active.lower() != '.true.'): - errmsg = "Non optional scheme arguments for conditionally allocatable variables" - sname = dict_var.get_prop_value('standard_name') - errmsg += ", {}".format(sname) - raise CCPPError(errmsg) - # end if - # Add the variable to the parent call tree - if dict_dims == new_dict_dims: - sdict = {} - else: - sdict = {'dimensions':new_dict_dims} - # end if - found_var = self.parent.add_variable_to_call_tree(var, - subst_dict=sdict) - if not match: - found_var = False - nctx = context_string(var.context) - nname = var.get_prop_value('local_name') - hctx = context_string(dict_var.context) - hname = dict_var.get_prop_value('local_name') - raise CCPPError(err.format(nname=nname, nctx=nctx, - hname=hname, hctx=hctx)) - # end if - # end if - # We have a match! - # Are the Scheme's and Host's compatible? - # If so, create compatibility object, containing any necessary - # forward/reverse transforms to/from and . - if dict_var is not None: - dict_var = self.parent.find_variable(source_var=var, any_scope=True) - compat_obj = var.compatible(dict_var, run_env) - # end if - return found_var, dict_var, var_vdim, new_vdims, compat_obj - - def part(self, index, error=True): - """Return one of this SuiteObject's parts raise an exception, or, - if is False, just return None""" - plen = len(self.__parts) - if (0 <= index < plen) or (abs(index) <= plen): - return self.__parts[index] - # end if - if error: - errmsg = 'No part {} in {} {}'.format(index, - self.__class__.__name__, - self.name) - raise ParseInternalError(errmsg) - # end if - return None - - def has_item(self, item_name): - """Return True iff item, , is already in this SuiteObject""" - has = False - for item in self.__parts: - if item.name == item_name: - has = True - else: - has = item.has_item(item_name) - # end if - if has: - break - # end if - # end for - return has - - @property - def name(self): - """Return the name of the element""" - return self.__name - - @name.setter - def name(self, value): - """Set the name of the element if it has not been set""" - if self.__name is None: - self.__name = value - else: - errmsg = 'Attempt to change name of {} to {}' - raise ParseInternalError(errmsg.format(self, value)) - # end if - - @property - def parent(self): - """This SuiteObject's parent (or none)""" - return self.__parent - - @property - def call_list(self): - """Return the SuiteObject's call_list""" - return self.__call_list - - @property - def phase_type(self): - """Return the phase_type of this suite_object""" - return self.__phase_type - - @property - def parts(self): - """Return a copy the component parts of this SuiteObject. - Returning a copy allows for the part list to be changed during - processing of the return value""" - return self.__parts[:] - - @property - def context(self): - """Return the context of this SuiteObject""" - return self.__context - - @property - def run_env(self): - """Return the CCPPFrameworkEnv runtime object for this SuiteObject""" - return self.__run_env - - def __repr__(self): - """Create a unique readable string for this Object""" - so_repr = super().__repr__() - olmatch = _OBJ_LOC_RE.search(so_repr) - if olmatch is not None: - loc = ' at {}'.format(olmatch.group(1)) - else: - loc = "" - # end if - return '<{} {}{}>'.format(self.__class__.__name__, self.name, loc) - - def __format__(self, spec): - """Return a string representing the SuiteObject, including its children. - is used between subitems. - is the indent level for multi-line output. - """ - if spec: - sep = spec[0] - else: - sep = '\n' - # end if - try: - ind_level = int(spec[1:]) - except (ValueError, IndexError): - ind_level = 0 - # end try - if sep == '\n': - indent = " " - else: - indent = "" - # end if - if self.name == self.__class__.__name__: - # This object does not have separate name - nstr = self.name - else: - nstr = "{}: {}".format(self.__class__.__name__, self.name) - # end if - output = "{}<{}>".format(indent*ind_level, nstr) - subspec = "{}{}".format(sep, ind_level + 1) - substr = "{o}{s}{p:" + subspec + "}" - subout = "" - for part in self.parts: - subout = substr.format(o=subout, s=sep, p=part) - # end for - if subout: - output = "{}{}{}{}".format(output, subout, sep, - indent*ind_level, - self.__class__.__name__) - else: - output = "{}".format(output, self.__class__.__name__) - # end if - return output - -############################################################################### - -class Scheme(SuiteObject): - """A single scheme in a suite (e.g., init method)""" - - def __init__(self, scheme_xml, context, parent, run_env): - """Initialize this physics Scheme""" - name = scheme_xml.text - self.__subroutine_name = None - self.__context = context - self.__version = scheme_xml.get('version', None) - self.__lib = scheme_xml.get('lib', None) - self.__has_vertical_dimension = False - self.__group = None - self.__forward_transforms = list() - self.__reverse_transforms = list() - self._has_run_phase = True - self.__optional_vars = list() - super().__init__(name, context, parent, run_env, active_call_list=True) - - def update_group_call_list_variable(self, var): - """If is in our group's call list, update its intent. - Add to our group's call list unless: - - is in our group's call list - - is in our group's dictionary, - - is a suite variable""" - stdname = var.get_prop_value('standard_name') - my_group = self.__group - gvar = my_group.call_list.find_variable(standard_name=stdname, - any_scope=False) - if gvar: - gvar.adjust_intent(var) - else: - gvar = my_group.find_variable(standard_name=stdname, - any_scope=False) - if gvar is None: - # Check for suite variable - gvar = my_group.find_variable(standard_name=stdname, - any_scope=True) - if gvar and (not SuiteObject.is_suite_variable(gvar)): - gvar = None - # end if - if gvar is None: - my_group.add_call_list_variable(var, gen_unique=True) - # end if - # end if - - def is_local_variable(self, var): - """Return None as we never consider to be in our local - dictionary. - This is an override of the SuiteObject version""" - return None - - def analyze(self, phase, group, scheme_library, suite_vars, level): - """Analyze the scheme's interface to prepare for writing""" - self.__group = group - my_header = None - if self.name in scheme_library: - func = scheme_library[self.name] - if phase in func: - my_header = func[phase] - self.__subroutine_name = my_header.title - else: - self._has_run_phase = False - return set() - # end if - else: - estr = 'No schemes found for {}' - raise ParseInternalError(estr.format(self.name), - context=self.__context) - # end if - if my_header is None: - estr = 'No {} header found for scheme, {}' - raise ParseInternalError(estr.format(phase, self.name), - context=self.__context) - # end if - if my_header.module is None: - estr = 'No module found for subroutine, {}' - raise ParseInternalError(estr.format(self.subroutine_name), - context=self.__context) - # end if - scheme_mods = set() - scheme_mods.add((my_header.module, self.subroutine_name)) - for var in my_header.variable_list(): - vstdname = var.get_prop_value('standard_name') - def_val = var.get_prop_value('default_value') - vdims = var.get_dimensions() - vintent = var.get_prop_value('intent') - args = self.match_variable(var, self.run_env) - found, dict_var, vert_dim, new_dims, compat_obj = args - if found: - # Hack to get the missing dimensions promoted to the right place - # Add variable allocation checks for group, suite and host variables - if dict_var: - self.handle_downstream_variables(dict_var) - # end if - if not self.has_vertical_dim: - self.__has_vertical_dimension = vert_dim is not None - # end if - # We have a match, make sure var is in call list - if new_dims == vdims: - self.add_call_list_variable(var, exists_ok=True, gen_unique=True) - self.update_group_call_list_variable(var) - else: - subst_dict = {'dimensions':new_dims} - clone = var.clone(subst_dict) - self.add_call_list_variable(clone, exists_ok=True) - self.update_group_call_list_variable(clone) - # end if - else: - if vintent == 'out': - if self.__group is None: - errmsg = 'Group not defined for {}'.format(self.name) - raise ParseInternalError(errmsg) - # end if - # The Group will manage this variable - self.__group.manage_variable(var) - self.add_call_list_variable(var) - elif def_val and (vintent != 'out'): - if self.__group is None: - errmsg = 'Group not defined for {}'.format(self.name) - raise ParseInternalError(errmsg) - # end if - # The Group will manage this variable - self.__group.manage_variable(var) - # We still need it in our call list (the group uses a clone) - self.add_call_list_variable(var) - else: - errmsg = 'Input argument for {}, {}, not found.' - if self.find_variable(source_var=var) is not None: - # The variable exists, maybe it is dim mismatch - lname = var.get_prop_value('local_name') - emsg = '\nCheck for dimension mismatch in {}' - errmsg += emsg.format(lname) - # end if - if ((not self.run_phase()) and - (vstdname in CCPP_LOOP_VAR_STDNAMES)): - emsg = '\nLoop variables not allowed in {} phase.' - errmsg += emsg.format(self.phase()) - # end if - raise CCPPError(errmsg.format(self.subroutine_name, - vstdname)) - # end if - # end if - # Are there any forward/reverse transforms for this variable? - has_transform = False - if compat_obj is not None and (compat_obj.has_vert_transforms or - compat_obj.has_unit_transforms or - compat_obj.has_kind_transforms): - self.add_var_transform(var, compat_obj, vert_dim) - has_transform = True - # end if - - # Is this a conditionally allocated variable? - # If so, declare localpointer variable. This is needed to - # pass inactive (not present) status through the caps. - if var.get_prop_value('optional'): - newvar_ptr = var.clone(var.get_prop_value('local_name')+'_ptr') - self.__optional_vars.append([dict_var, var, newvar_ptr, has_transform]) - # end if - - # end for - return scheme_mods - - def handle_downstream_variables(self, var): - """Ensure all dimension and optional variable arguments are available""" - # Get the basic attributes that decide whether we need - # to check the variable when we write the group - standard_name = var.get_prop_value('standard_name') - dimensions = var.get_dimensions() - active = var.get_prop_value('active') - var_dicts = [ self.__group.call_list ] + self.__group.suite_dicts() - - # If the variable isn't active, skip it - if active.lower() =='.false.': - return - # Also, if the variable is one of the CCPP error handling messages, skip it - # since it is defined as intent(out) and we can't do meaningful checks on it - elif standard_name == 'ccpp_error_code' or standard_name == 'ccpp_error_message': - return - # To perform allocation checks, we need to know all variables - # that are part of the 'active' attribute conditional and add - # it to the group's call list. - else: - (_, vars_needed) = var.conditional(var_dicts) - for var_needed in vars_needed: - self.update_group_call_list_variable(var_needed) - - # For arrays, we need to get information on the dimensions and add it to - # the group's call list so that we can test for the correct size later on - if dimensions: - for dim in dimensions: - if not ':' in dim: - dim_var = self.find_variable(standard_name=dim) - if not dim_var: - # To allow for numerical dimensions in metadata. - if not dim.isnumeric(): - raise Exception(f"No dimension with standard name '{dim}'") - # end if - else: - self.update_group_call_list_variable(dim_var) - # end if - else: - (ldim, udim) = dim.split(":") - ldim_var = self.find_variable(standard_name=ldim) - if not ldim_var: - # To allow for numerical dimensions in metadata. - if not ldim.isnumeric(): - raise Exception(f"No dimension with standard name '{ldim}'") - # end if - # end if - self.update_group_call_list_variable(ldim_var) - udim_var = self.find_variable(standard_name=udim) - if not udim_var: - # To allow for numerical dimensions in metadata. - if not udim.isnumeric(): - raise Exception(f"No dimension with standard name '{udim}'") - # end if - else: - self.update_group_call_list_variable(udim_var) - # end if - - def associate_optional_var(self, dict_var, var, var_ptr, has_transform, cldicts, indent, outfile): - """Write local pointer association for optional variables.""" - if (dict_var): - (conditional, _) = dict_var.conditional(cldicts) - if (has_transform): - lname = var.get_prop_value('local_name')+'_local' - else: - lname = var.get_prop_value('local_name') - # end if - lname_ptr = var_ptr.get_prop_value('local_name') - outfile.write(f"if {conditional} then", indent) - outfile.write(f"{lname_ptr} => {lname}", indent+1) - outfile.write(f"end if", indent) - # end if - - def assign_pointer_to_var(self, dict_var, var, var_ptr, has_transform, cldicts, indent, outfile): - """Assign local pointer to variable.""" - if (dict_var): - intent = var.get_prop_value('intent') - if (intent == 'out' or intent == 'inout'): - (conditional, _) = dict_var.conditional(cldicts) - if (has_transform): - lname = var.get_prop_value('local_name')+'_local' - else: - lname = var.get_prop_value('local_name') - # end if - lname_ptr = var_ptr.get_prop_value('local_name') - outfile.write(f"if {conditional} then", indent) - outfile.write(f"{lname} = {lname_ptr}", indent+1) - outfile.write(f"end if", indent) - # end if - # end if - - def add_var_transform(self, var, compat_obj, vert_dim): - """Register any variable transformation needed by for this Scheme. - For any transformation identified in , create dummy variable - from to perform the transformation. Determine the indices needed - for the transform and save for use during write stage""" - - # Add local variable (_local) needed for transformation. - # Do not let the Group manage this variable. Handle local var - # when writing Group. - prop_dict = var.copy_prop_dict() - prop_dict['local_name'] = var.get_prop_value('local_name')+'_local' - # This is a local variable. - if 'intent' in prop_dict: - del prop_dict['intent'] - # end if - local_trans_var = Var(prop_dict, - ParseSource(_API_SOURCE_NAME, - _API_LOCAL_VAR_NAME, var.context), - self.run_env) - found = self.__group.find_variable(source_var=local_trans_var, any_scope=False) - if not found: - lmsg = "Adding new local variable, '{}', for variable transform" - self.run_env.logger.info(lmsg.format(local_trans_var.get_prop_value('local_name'))) - self.__group.transform_locals.append(local_trans_var) - # end if - - # Create indices (default) for transform. - lindices = [':']*var.get_rank() - rindices = [':']*var.get_rank() - - # If needed, modify vertical dimension for vertical orientation flipping - _, vdim = find_vertical_dimension(var.get_dimensions()) - if vdim >= 0: - vdims = vert_dim.split(':') - vdim_name = vdims[-1] - group_vvar = self.__group.call_list.find_variable(vdim_name) - if group_vvar is None: - raise CCPPError(f"add_var_transform: Cannot find dimension variable, {vdim_name}") - # end if - vname = group_vvar.get_prop_value('local_name') - if len(vdims) == 2: - sdim_name = vdims[0] - group_vvar = self.find_variable(sdim_name) - if group_vvar is None: - raise CCPPError(f"add_var_transform: Cannot find dimension variable, {sdim_name}") - # end if - sname = group_vvar.get_prop_value('local_name') - else: - sname = '1' - # end if - lindices[vdim] = sname+':'+vname - if compat_obj.has_vert_transforms: - rindices[vdim] = vname+':'+sname+':-1' - else: - rindices[vdim] = sname+':'+vname - # end if - # end if - - # If needed, modify horizontal dimension for loop substitution. - # NOT YET IMPLEMENTED - #hdim = find_horizontal_dimension(var.get_dimensions()) - #if compat_obj.has_dim_transforms: - - # Register any reverse (pre-Scheme) transforms. Also, save local_name used in - # transform (used in write stage). - if (var.get_prop_value('intent') != 'out'): - lmsg = "Automatic unit conversion from '{}' to '{}' for '{}' before entering '{}'" - self.run_env.logger.info(lmsg.format(compat_obj.v2_units, - compat_obj.v1_units, - compat_obj.v2_stdname, - self.__subroutine_name)) - self.__reverse_transforms.append([local_trans_var.get_prop_value('local_name'), - var.get_prop_value('local_name'), - var.get_prop_value('standard_name'), - rindices, lindices, compat_obj]) - # end if - # Register any forward (post-Scheme) transforms. - if (var.get_prop_value('intent') != 'in'): - lmsg = "Automatic unit conversion from '{}' to '{}' for '{}' after returning '{}'" - self.run_env.logger.info(lmsg.format(compat_obj.v1_units, - compat_obj.v2_units, - compat_obj.v1_stdname, - self.__subroutine_name)) - self.__forward_transforms.append([var.get_prop_value('local_name'), - var.get_prop_value('standard_name'), - local_trans_var.get_prop_value('local_name'), - lindices, rindices, compat_obj]) - # end if - def write_var_transform(self, var, dummy, rindices, lindices, compat_obj, - outfile, indent, forward): - """Write variable transformation needed to call this Scheme in . - is the variable that needs transformation before and after calling Scheme. - is the local variable needed for the transformation.. - are the LHS indices of for reverse transforms (before Scheme). - are the RHS indices of for reverse transforms (before Scheme). - are the LHS indices of for forward transforms (after Scheme). - are the RHS indices of for forward transforms (after Scheme). - """ - # - # Write reverse (pre-Scheme) transform. - # - if not forward: - # dummy(lindices) = var(rindices) - stmt = compat_obj.reverse_transform(lvar_lname=dummy, - rvar_lname=var, - lvar_indices=lindices, - rvar_indices=rindices) - # - # Write forward (post-Scheme) transform. - # - else: - # var(lindices) = dummy(rindices) - stmt = compat_obj.forward_transform(lvar_lname=var, - rvar_lname=dummy, - lvar_indices=rindices, - rvar_indices=lindices) - # end if - outfile.write(stmt, indent) - - def write(self, outfile, errcode, errmsg, indent): - # Unused arguments are for consistent write interface - # pylint: disable=unused-argument - """Write code to call this Scheme to """ - # Dictionaries to try are our group, the group's call list, - # or our module - cldicts = [self.__group, self.__group.call_list] - cldicts.extend(self.__group.suite_dicts()) - my_args = self.call_list.call_string(cldicts=cldicts, - is_func_call=True, - subname=self.subroutine_name, - sub_lname_list = self.__reverse_transforms) - # - outfile.write('', indent) - outfile.write('if ({} == 0) then'.format(errcode), indent) - # - # Write any reverse (pre-Scheme) transforms. - if len(self.__reverse_transforms) > 0: - outfile.comment('Compute reverse (pre-scheme) transforms', indent+1) - # end if - for rcnt, (dummy, var_lname, var_sname, rindices, lindices, compat_obj) in enumerate(self.__reverse_transforms): - # Any transform(s) were added during the Group's analyze phase, but - # the local_name(s) of the assoicated with the transform(s) - # may have since changed. Here we need to use the standard_name - # from and replace its local_name with the local_name from the - # Group's call_list. - lvar = self.__group.call_list.find_variable(standard_name=var_sname) - lvar_lname = lvar.get_prop_value('local_name') - tstmt = self.write_var_transform(lvar_lname, dummy, rindices, lindices, compat_obj, outfile, indent+1, False) - # end for - outfile.write('',indent+1) - # - # Associate any conditionally allocated variables. - # - if self.__optional_vars: - outfile.write('! Associate conditional variables', indent+1) - # end if - for (dict_var, var, var_ptr, has_transform) in self.__optional_vars: - tstmt = self.associate_optional_var(dict_var, var, var_ptr, has_transform, cldicts, indent+1, outfile) - # end for - # - # Write the scheme call. - # - if self._has_run_phase: - stmt = 'call {}({})' - outfile.write('',indent+1) - outfile.write('! Call scheme', indent+1) - outfile.write(stmt.format(self.subroutine_name, my_args), indent+1) - outfile.write('',indent+1) - # end if - # - # Copy any local pointers. - # - first_ptr_declaration=True - for (dict_var, var, var_ptr, has_transform) in self.__optional_vars: - if first_ptr_declaration: - outfile.write('! Copy any local pointers to dummy/local variables', indent+1) - first_ptr_declaration=False - # end if - tstmt = self.assign_pointer_to_var(dict_var, var, var_ptr, has_transform, cldicts, indent+1, outfile) - # end for - outfile.write('',indent+1) - # - # Write any forward (post-Scheme) transforms. - # - if len(self.__forward_transforms) > 0: - outfile.comment('Compute forward (post-scheme) transforms', indent+1) - # end if - for fcnt, (var_lname, var_sname, dummy, lindices, rindices, compat_obj) in enumerate(self.__forward_transforms): - # Any transform(s) were added during the Group's analyze phase, but - # the local_name(s) of the assoicated with the transform(s) - # may have since changed. Here we need to use the standard_name - # from and replace its local_name with the local_name from the - # Group's call_list. - lvar = self.__group.call_list.find_variable(standard_name=var_sname) - lvar_lname = lvar.get_prop_value('local_name') - tstmt = self.write_var_transform(lvar_lname, dummy, rindices, lindices, compat_obj, outfile, indent+1, True) - # end for - outfile.write('', indent) - outfile.write('end if', indent) - - def schemes(self): - """Return self as a list for consistency with subcycle""" - return [self] - - def variable_list(self, recursive=False, - std_vars=True, loop_vars=True, consts=True): - """Return a list of all variables for this Scheme. - Because Schemes do not have any variables, return a list - of this object's CallList variables instead. - Note that because of this, is not allowed.""" - if recursive: - raise ParseInternalError("recursive=True not allowed for Schemes") - # end if - return self.call_list.variable_list(recursive=recursive, - std_vars=std_vars, - loop_vars=loop_vars, consts=consts) - - @property - def subroutine_name(self): - """Return this scheme's actual subroutine name""" - return self.__subroutine_name - - @property - def has_vertical_dim(self): - """Return True if at least one of this Scheme's variables has - a vertical dimension (vertical_layer_dimension or - vertical_interface_dimension) - """ - return self.__has_vertical_dimension - - def __str__(self): - """Create a readable string for this Scheme""" - return ''.format(self.name, self.subroutine_name) - -############################################################################### - -class Subcycle(SuiteObject): - """Class to represent a subcycled group of schemes or scheme collections""" - - def __init__(self, sub_xml, context, parent, run_env, loop_count=0): - self._loop_extent = sub_xml.get('loop', "1") # Number of iterations - self._loop = None - # See if our loop variable is an integer or a variable - try: - _ = int(self._loop_extent) - self._loop = self._loop_extent - self._loop_var_int = True - name = f"loop{loop_count}" - super().__init__(name, context, parent, run_env, active_call_list=False) - loop_count = loop_count + 1 - except ValueError: - self._loop_var_int = False - lvar = parent.find_variable(standard_name=self._loop_extent, any_scope=True) - if lvar is None: - emsg = "Subcycle, {}, specifies {} iterations, variable not found" - raise CCPPError(emsg.format(name, self._loop_extent)) - else: - self._loop_var_int = False - self._loop = lvar.get_prop_value('local_name') - # end if - name = f"loop{loop_count}_{self._loop_extent}"[0:63] - super().__init__(name, context, parent, run_env, active_call_list=True) - parent.add_call_list_variable(lvar, exists_ok=True) - loop_count = loop_count + 1 - # end try - for item in sub_xml: - new_item = new_suite_object(item, context, self, run_env, loop_count=loop_count) - self.add_part(new_item) - # end for - - def analyze(self, phase, group, scheme_library, suite_vars, level): - """Analyze the Subcycle's interface to prepare for writing""" - if self.name is None: - self.name = "subcycle_index{}".format(level) - # end if - # Create a Group variable for the subcycle index. - newvar = Var({'local_name':self.name, 'standard_name':self.name, - 'type':'integer', 'units':'count', 'dimensions':'()'}, - _API_LOCAL, self.run_env) - group.manage_variable(newvar) - # Handle all the suite objects inside of this subcycle - scheme_mods = set() - for item in self.parts: - smods = item.analyze(phase, group, scheme_library, - suite_vars, level+1) - for smod in smods: - scheme_mods.add(smod) - # end for - # end for - return scheme_mods - - def write(self, outfile, errcode, errmsg, indent): - """Write code for the subcycle loop, including contents, to """ - outfile.write('do {} = 1, {}'.format(self.name, self._loop), indent) - # Note that 'scheme' may be a sybcycle or other construct - for item in self.parts: - item.write(outfile, errcode, errmsg, indent+1) - # end for - outfile.write('end do', 2) - - @property - def loop(self): - """Return the loop value or variable local_name""" - return self._loop - -############################################################################### - -class Group(SuiteObject): - """Class to represent a grouping of schemes in a suite - A Group object is implemented as a subroutine callable by the API. - The main arguments to a group are the host model variables. - Additional output arguments are generated from schemes with intent(out) - arguments. - Additional input or inout arguments are generated for inputs needed by - schemes which are produced (intent(out)) by other groups. - """ - - __subhead = ''' - subroutine {subname}({args}) -''' - - __subend = ''' - end subroutine {subname} - -! ======================================================================== -''' - - __thread_check = CodeBlock([('#ifdef _OPENMP', -1), - ('if (omp_get_thread_num() > 1) then', 1), - ('{errcode} = 1', 2), - (('{errmsg} = "Cannot call {phase} routine ' - 'from a threaded region"'), 2), - ('return', 2), - ('end if', 1), - ('#endif', -1)]) - - - def __init__(self, group_xml, transition, parent, context, run_env): - """Initialize this Group object from . - is the group's phase, is the group's suite. - """ - name = parent.name + '_' + group_xml.get('name') - if transition not in CCPP_STATE_MACH.transitions(): - errmsg = "Bad transition argument to Group, '{}'" - raise ParseInternalError(errmsg.format(transition)) - # end if - # Initialize the dictionary of variables internal to group - super().__init__(name, context, parent, run_env, - active_call_list=True, phase_type=transition) - add_to = self - # Add the sub objects - for item in group_xml: - new_item = new_suite_object(item, context, add_to, run_env) - add_to.add_part(new_item) - # end for - self._local_schemes = set() - self._host_vars = None - self._host_ddts = None - self._loop_var_matches = list() - self._phase_check_stmts = list() - self._set_state = None - self._ddt_library = None - self.transform_locals = list() - - def phase_match(self, scheme_name): - """If scheme_name matches the group phase, return the group and - function ID. Otherwise, return None - """ - fid, tid, _ = CCPP_STATE_MACH.transition_match(scheme_name, - transition=self.phase()) - if tid is not None: - return self, fid - # end if - return None, None - - def move_to_call_list(self, standard_name): - """Move a variable from the group internal dictionary to the call list. - This is done when the variable, , will be allocated by - the suite. - """ - gvar = self.find_variable(standard_name=standard_name, any_scope=False) - if gvar is None: - errmsg = "Group {}, cannot move {}, variable not found" - raise ParseInternalError(errmsg.format(self.name, standard_name)) - # end if - self.add_call_list_variable(gvar, exists_ok=True) - self.remove_variable(standard_name) - - def register_action(self, vaction): - """Register any recognized type for use during self.write. - Return True iff is handled. - """ - if isinstance(vaction, VarLoopSubst): - self._loop_var_matches = vaction.add_to_list(self._loop_var_matches) - # Add the missing dim - vaction.add_local(self, _API_LOCAL, self.run_env) - return True - # end if - return False - - def manage_variable(self, newvar): - """Add to our local dictionary making necessary - modifications to the variable properties so that it is - allocated appropriately""" - # Need new prop dict to eliminate unwanted properties (e.g., intent) - vdims = newvar.get_dimensions() - # Look for dimensions where we have a loop substitution and replace - # with the correct size - if self.run_phase(): - hdims = [x.missing_stdname for x in self._loop_var_matches] - else: - # Do not do loop substitutions in full phases - hdims = list() - # end if - for index, dim in enumerate(vdims): - newdim = None - for subdim in dim.split(':'): - if subdim in hdims: - # We have a loop substitution, find and replace - hindex = hdims.index(subdim) - names = self._loop_var_matches[hindex].required_stdnames - newdim = ':'.join(names) - break - # end if - if ('vertical' in subdim) and ('index' in subdim): - # We have a vertical index, replace with correct dimension - errmsg = "vertical index replace not implemented" - raise ParseInternalError(errmsg) - # end if - # end for - if newdim is not None: - vdims[index] = newdim - # end if - # end for - if self.timestep_phase(): - persist = 'timestep' - else: - persist = 'run' - # end if - # Start with an official copy of 's prop_dict with - # corrected dimensions - subst_dict = {'dimensions':vdims} - prop_dict = newvar.copy_prop_dict(subst_dict=subst_dict) - # Add the allocatable items - prop_dict['allocatable'] = len(vdims) > 0 # No need to allocate scalar - prop_dict['persistence'] = persist - # This is a local variable - if 'intent' in prop_dict: - del prop_dict['intent'] - # end if - # Create a new variable, save the original context - local_var = Var(prop_dict, - ParseSource(_API_SOURCE_NAME, - _API_LOCAL_VAR_NAME, newvar.context), - self.run_env) - self.add_variable(local_var, self.run_env, exists_ok=True, gen_unique=True) - # Finally, make sure all dimensions are accounted for - emsg = self.add_variable_dimensions(local_var, [_API_LOCAL_VAR_NAME], - _API_SUITE_VAR_NAME, - adjust_intent=True, - to_dict=self.call_list) - if emsg: - raise CCPPError(emsg) - # end if - - def analyze(self, phase, suite_vars, scheme_library, ddt_library, - check_suite_state, set_suite_state): - """Analyze the Group's interface to prepare for writing""" - self._ddt_library = ddt_library - # Sanity check for Group - if phase != self.phase(): - errmsg = 'Group {} has phase {} but analyze is phase {}' - raise ParseInternalError(errmsg.format(self.name, - self.phase(), phase)) - # end if - for item in self.parts: - # Items can be schemes, subcycles or other objects - # All have the same interface and return a set of module use - # statements (lschemes) - lschemes = item.analyze(phase, self, scheme_library, suite_vars, 1) - for lscheme in lschemes: - self._local_schemes.add(lscheme) - # end for - # end for - self._phase_check_stmts = check_suite_state - self._set_state = set_suite_state - if (self.run_env.logger and - self.run_env.logger.isEnabledFor(logging.DEBUG)): - self.run_env.logger.debug("{}".format(self)) - # end if - - def allocate_dim_str(self, dims, context): - """Create the dimension string for an allocate statement""" - rdims = list() - for dim in dims: - rdparts = list() - dparts = dim.split(':') - for dpart in dparts: - dvar = self.find_variable(standard_name=dpart, any_scope=False) - if dvar is None: - dvar = self.call_list.find_variable(standard_name=dpart, - any_scope=False) - # end if - if dvar is None: - # Check if it's a module-level variable - dvar = self.find_variable(standard_name=dpart, any_scope=True) - # end if - if dvar is None: - emsg = "Dimension variable, '{}', not found{}" - lvar = self.find_local_name(dpart, any_scope=True) - if lvar is not None: - emsg += "\nBe sure to use standard names!" - # end if - ctx = context_string(context) - raise CCPPError(emsg.format(dpart, ctx)) - # end if - lname = dvar.get_prop_value('local_name') - rdparts.append(lname) - # end for - rdims.append(':'.join(rdparts)) - # end for - return ', '.join(rdims) - - def find_variable(self, standard_name=None, source_var=None, - any_scope=True, clone=None, - search_call_list=False, loop_subst=False): - """Find a matching variable to , create a local clone (if - is True), or return None. - This purpose of this special Group version is to record any constituent - variable found for processing during the write phase. - """ - fvar = super().find_variable(standard_name=standard_name, - source_var=source_var, - any_scope=any_scope, clone=clone, - search_call_list=search_call_list, - loop_subst=loop_subst) - if fvar and fvar.is_constituent(): - if fvar.source.ptype == ConstituentVarDict.constitutent_source_type(): - # We found this variable in the constituent dictionary, - # add it to our call list - self.add_call_list_variable(fvar, exists_ok=True) - # end if - # end if - return fvar - - def write(self, outfile, host_arglist, indent, const_mod, - suite_vars=None, allocate=False, deallocate=False): - """Write code for this subroutine (Group), including contents, - to """ - # Unused arguments are for consistent write interface - # pylint: disable=unused-argument - # group type for (de)allocation - if self.timestep_phase(): - group_type = 'timestep' # Just allocate for the timestep - else: - group_type = 'run' # Allocate for entire run - # end if - # Collect information on local variables - subpart_allocate_vars = {} - subpart_optional_vars = {} - subpart_scalar_vars = {} - allocatable_var_set = set() - optional_var_set = set() - pointer_var_set = list() - inactive_var_set = set() - for item in [self]:# + self.parts: - for var in item.declarations(): - lname = var.get_prop_value('local_name') - sname = var.get_prop_value('standard_name') - if (lname in subpart_allocate_vars) or (lname in subpart_optional_vars) or (lname in subpart_scalar_vars): - if subpart_allocate_vars[lname][0].compatible(var, self.run_env): - pass # We already are going to declare this variable - else: - errmsg = "Duplicate Group variable, {}" - raise ParseInternalError(errmsg.format(lname)) - # end if - else: - opt_var = var.get_prop_value('optional') - dims = var.get_dimensions() - if (dims is not None) and dims: - if opt_var: - if (self.call_list.find_variable(standard_name=sname)): - subpart_optional_vars[lname] = (var, item, opt_var) - optional_var_set.add(lname) - else: - inactive_var_set.add(var) - # end if - else: - subpart_allocate_vars[lname] = (var, item, opt_var) - allocatable_var_set.add(lname) - # end if - else: - subpart_scalar_vars[lname] = (var, item, opt_var) - # end if - # end if - # end for - # All optional dummy variables within group need to have - # an associated pointer array declared. - for cvar in self.call_list.variable_list(): - opt_var = cvar.get_prop_value('optional') - if opt_var: - name = cvar.get_prop_value('local_name')+'_ptr' - kind = cvar.get_prop_value('kind') - dims = cvar.get_dimensions() - if cvar.is_ddt(): - vtype = 'type' - else: - vtype = cvar.get_prop_value('type') - # end if - if dims: - dimstr = '(:' + ',:'*(len(dims) - 1) + ')' - else: - dimstr = '' - # end if - pointer_var_set.append([name,kind,dimstr,vtype]) - # end if - # end for - # Any optional arguments that are not requested by the host need to have - # a local null pointer passed from the group to the scheme. - for ivar in inactive_var_set: - name = ivar.get_prop_value('local_name')+'_ptr' - kind = ivar.get_prop_value('kind') - dims = ivar.get_dimensions() - if ivar.is_ddt(): - vtype = 'type' - else: - vtype = ivar.get_prop_value('type') - # end if - if dims: - dimstr = '(:' + ',:'*(len(dims) - 1) + ')' - else: - dimstr = '' - # end if - pointer_var_set.append([name,kind,dimstr,vtype]) - # end for - # Any arguments used in variable transforms before or after the - # Scheme call? If so, declare local copy for reuse in the Group cap. - for ivar in self.transform_locals: - lname = ivar.get_prop_value('local_name') - opt_var = ivar.get_prop_value('optional') - dims = ivar.get_dimensions() - if (dims is not None) and dims: - subpart_allocate_vars[lname] = (ivar, item, opt_var) - allocatable_var_set.add(lname) - else: - subpart_scalar_vars[lname] = (ivar, item, opt_var) - # end if - # end for - - # end for - # First, write out the subroutine header - subname = self.name - call_list = self.call_list.call_string() - outfile.write(Group.__subhead.format(subname=subname, args=call_list), - indent) - # Write out any use statements - if self._local_schemes: - modmax = max([len(s[0]) for s in self._local_schemes]) - else: - modmax = 0 - # end if - # Write out the scheme use statements - scheme_use = 'use {},{} only: {}' - for scheme in sorted(self._local_schemes): - smod = scheme[0] - sname = scheme[1] - slen = ' '*(modmax - len(smod)) - outfile.write(scheme_use.format(smod, slen, sname), indent+1) - # end for - # Look for any DDT types - call_vars = self.call_list.variable_list() - all_vars = ([x[0] for x in subpart_allocate_vars.values()] + - [x[0] for x in subpart_scalar_vars.values()] + - [x[0] for x in subpart_optional_vars.values()]) - all_vars.extend(call_vars) - self._ddt_library.write_ddt_use_statements(all_vars, outfile, - indent+1, pad=modmax) - outfile.write('', 0) - # Write out dummy arguments - outfile.write('! Dummy arguments', indent+1) - msg = 'Variables for {}: ({})' - if (self.run_env.logger and - self.run_env.logger.isEnabledFor(logging.DEBUG)): - self.run_env.logger.debug(msg.format(self.name, call_vars)) - # end if - self.call_list.declare_variables(outfile, indent+1, dummy=True) - # DECLARE local variables - if subpart_allocate_vars or subpart_scalar_vars or subpart_optional_vars: - outfile.write('\n! Local Variables', indent+1) - # end if - # Scalars - for key in subpart_scalar_vars: - var = subpart_scalar_vars[key][0] - spdict = subpart_scalar_vars[key][1] - target = subpart_scalar_vars[key][2] - var.write_def(outfile, indent+1, spdict, - allocatable=False, target=target) - # end for - # Allocatable arrays - for key in subpart_allocate_vars: - var = subpart_allocate_vars[key][0] - spdict = subpart_allocate_vars[key][1] - target = subpart_allocate_vars[key][2] - var.write_def(outfile, indent+1, spdict, - allocatable=(key in allocatable_var_set), - target=target) - # end for - # Target arrays. - for key in subpart_optional_vars: - var = subpart_optional_vars[key][0] - spdict = subpart_optional_vars[key][1] - target = subpart_optional_vars[key][2] - var.write_def(outfile, indent+1, spdict, - allocatable=(key in optional_var_set), - target=target) - # end for - # Pointer variables - for (name, kind, dim, vtype) in pointer_var_set: - var.write_ptr_def(outfile, indent+1, name, kind, dim, vtype) - # end for - outfile.write('', 0) - # Get error variable names - if self.run_env.use_error_obj: - raise ParseInternalError("Error object not supported") - else: - verrcode = self.call_list.find_variable(standard_name='ccpp_error_code') - if verrcode is not None: - errcode = verrcode.get_prop_value('local_name') - else: - errmsg = "No ccpp_error_code variable for group, {}" - raise CCPPError(errmsg.format(self.name)) - # end if - verrmsg = self.call_list.find_variable(standard_name='ccpp_error_message') - if verrmsg is not None: - errmsg = verrmsg.get_prop_value('local_name') - else: - errmsg = "No ccpp_error_message variable for group, {}" - raise CCPPError(errmsg.format(self.name)) - # end if - # Initialize error variables - outfile.write("! Initialize ccpp error handling", 2) - outfile.write("{} = 0".format(errcode), 2) - outfile.write("{} = ''".format(errmsg), 2) - outfile.write("",2) - # end if - # Output threaded region check (except for run phase) - if not self.run_phase(): - outfile.write("! Output threaded region check ",indent+1) - Group.__thread_check.write(outfile, indent, - {'phase' : self.phase(), - 'errcode' : errcode, - 'errmsg' : errmsg}) - # Check state machine - outfile.write("! Check state machine",indent+1) - self._phase_check_stmts.write(outfile, indent, - {'errcode' : errcode, 'errmsg' : errmsg, - 'funcname' : self.name}) - # Write any loop match calculations - outfile.write("! Set horizontal loop extent",indent+1) - for vmatch in self._loop_var_matches: - action = vmatch.write_action(self, dict2=self.call_list) - if action: - outfile.write(action, indent+1) - # end if - # end for - # Allocate local arrays - outfile.write('\n! Allocate local arrays', indent+1) - alloc_stmt = "allocate({}({}))" - for lname in sorted(allocatable_var_set): - var = subpart_allocate_vars[lname][0] - dims = var.get_dimensions() - alloc_str = self.allocate_dim_str(dims, var.context) - outfile.write(alloc_stmt.format(lname, alloc_str), indent+1) - # end for - for lname in optional_var_set: - var = subpart_optional_vars[lname][0] - dims = var.get_dimensions() - alloc_str = self.allocate_dim_str(dims, var.context) - outfile.write(alloc_stmt.format(lname, alloc_str), indent+1) - # end for - # Allocate suite vars - if allocate: - outfile.write('\n! Allocate suite_vars', indent+1) - for svar in suite_vars.variable_list(): - dims = svar.get_dimensions() - if dims: - timestep_var = svar.get_prop_value('persistence') - if group_type == timestep_var: - alloc_str = self.allocate_dim_str(dims, svar.context) - lname = svar.get_prop_value('local_name') - outfile.write(alloc_stmt.format(lname, alloc_str), - indent+1) - # end if (do not allocate in this phase) - # end if dims (do not allocate scalars) - # end for - # end if - # Write the scheme and subcycle calls - for item in self.parts: - item.write(outfile, errcode, errmsg, indent + 1) - # end for - # Deallocate local arrays - if allocatable_var_set: - outfile.write('\n! Deallocate local arrays', indent+1) - # end if - for lname in sorted(allocatable_var_set): - outfile.write('if (allocated({})) {} deallocate({})'.format(lname,' '*(20-len(lname)),lname), indent+1) - # end for - for lname in optional_var_set: - outfile.write('if (allocated({})) {} deallocate({})'.format(lname,' '*(20-len(lname)),lname), indent+1) - # end for - # Nullify local pointers - if pointer_var_set: - outfile.write('\n! Nullify local pointers', indent+1) - # end if - for (name, kind, dim, vtype) in pointer_var_set: - #cspace = ' '*(15-len(name)) - outfile.write('if (associated({})) {} nullify({})'.format(name,' '*(15-len(name)),name), indent+1) - # end fo - # Deallocate suite vars - if deallocate: - for svar in suite_vars.variable_list(): - dims = svar.get_dimensions() - if dims: - timestep_var = svar.get_prop_value('persistence') - if group_type == timestep_var: - lname = svar.get_prop_value('local_name') - outfile.write('deallocate({})'.format(lname), indent+1) - # end if - # end if (no else, do not deallocate scalars) - # end for - # end if - self._set_state.write(outfile, indent, {}) - # end if - outfile.write(Group.__subend.format(subname=subname), indent) - - @property - def suite(self): - """Return this Group's suite""" - return self.parent - - def suite_dicts(self): - """Return a list of this Group's Suite's dictionaries""" - return self.suite.suite_dicts() - -############################################################################### - -if __name__ == "__main__": - # First, run doctest - # pylint: disable=ungrouped-imports - import doctest - import sys - # pylint: enable=ungrouped-imports - fail, _ = doctest.testmod() - sys.exit(fail) -# end if diff --git a/scripts/var_props.py b/scripts/var_props.py deleted file mode 100755 index d732ae5e..00000000 --- a/scripts/var_props.py +++ /dev/null @@ -1,1565 +0,0 @@ -#!/usr/bin/env python3 - -""" -Classes and supporting code to hold all information on the compatibility of -two CCPP metadata variables. -VariableProperty: Class which describes a single variable property -VarCompatObj -""" - -# Python library imports -import keyword -import re -# CCPP framework imports -from conversion_tools import unit_conversion -from framework_env import CCPPFrameworkEnv -from parse_tools import check_local_name, check_fortran_type, context_string -from parse_tools import check_molar_mass -from parse_tools import FORTRAN_DP_RE, FORTRAN_SCALAR_REF_RE, fortran_list_match -from parse_tools import check_units, check_dimensions, check_cf_standard_name -from parse_tools import check_diagnostic_id, check_diagnostic_fixed -from parse_tools import check_default_value, check_valid_values -from parse_tools import ParseContext, ParseSource -from parse_tools import ParseInternalError, ParseSyntaxError, CCPPError - -############################################################################### -_REAL_SUBST_RE = re.compile(r"(.*\d)p(\d.*)") -_HDIM_TEMPNAME = '_CCPP_HORIZ_DIM' - -############################################################################### -# Supported horizontal dimensions (should be defined in CCPP_STANDARD_VARS) -CCPP_HORIZONTAL_DIMENSIONS = ['ccpp_constant_one:horizontal_dimension', - 'ccpp_constant_one:horizontal_loop_extent', - 'horizontal_loop_begin:horizontal_loop_end', - 'horizontal_dimension', 'horizontal_loop_extent'] - -############################################################################### -# Supported vertical dimensions (should be defined in CCPP_STANDARD_VARS) -CCPP_VERTICAL_DIMENSIONS = ['ccpp_constant_one:vertical_layer_dimension', - 'ccpp_constant_one:vertical_interface_dimension', - 'vertical_layer_dimension', - 'vertical_interface_dimension', - 'vertical_layer_index', 'vertical_interface_index'] - -############################################################################### -# Substituions for run time dimension control -CCPP_LOOP_DIM_SUBSTS = {'ccpp_constant_one:horizontal_dimension' : - 'horizontal_loop_begin:horizontal_loop_end', - 'ccpp_constant_one:vertical_layer_dimension' : - 'vertical_layer_index', - 'ccpp_constant_one:vertical_interface_dimension' : - 'vertical_interface_index'} - -######################################################################## -def is_horizontal_dimension(dim_name): -######################################################################## - """Return True if it is a recognized horizontal - dimension or index, otherwise, return False - >>> is_horizontal_dimension('horizontal_loop_extent') - True - >>> is_horizontal_dimension('ccpp_constant_one:horizontal_loop_extent') - True - >>> is_horizontal_dimension('ccpp_constant_one:horizontal_dimension') - True - >>> is_horizontal_dimension('horizontal_loop_begin:horizontal_loop_end') - True - >>> is_horizontal_dimension('horizontal_loop_begin:horizontal_loop_extent') - False - >>> is_horizontal_dimension('ccpp_constant_one') - False - """ - return dim_name in CCPP_HORIZONTAL_DIMENSIONS - -######################################################################## -def is_vertical_dimension(dim_name): -######################################################################## - """Return True if it is a recognized vertical - dimension or index, otherwise, return False - >>> is_vertical_dimension('ccpp_constant_one:vertical_layer_dimension') - True - >>> is_vertical_dimension('ccpp_constant_one:vertical_interface_dimension') - True - >>> is_vertical_dimension('vertical_layer_index') - True - >>> is_vertical_dimension('vertical_interface_index') - True - >>> is_vertical_dimension('ccpp_constant_one:vertical_layer_index') - False - >>> is_vertical_dimension('ccpp_constant_one:vertical_interface_index') - False - >>> is_vertical_dimension('horizontal_loop_extent') - False - """ - return dim_name in CCPP_VERTICAL_DIMENSIONS - -######################################################################## -def find_horizontal_dimension(dims): -######################################################################## - """Return the horizontal dimension string and location in - or (None, -1). - Return form is (horizontal_dimension, index) where index is the - location of horizontal_dimension in """ - var_hdim = None - hindex = -1 - for index, dimname in enumerate(dims): - if is_horizontal_dimension(dimname): - var_hdim = dimname - hindex = index - break - # end if - # end for - return (var_hdim, hindex) - -######################################################################## -def find_vertical_dimension(dims): -######################################################################## - """Return the vertical dimension string and location in - or (None, -1). - Return form is (vertical_dimension, index) where index is the - location of vertical_dimension in """ - var_vdim = None - vindex = -1 - for index, dimname in enumerate(dims): - if is_vertical_dimension(dimname): - var_vdim = dimname - vindex = index - break - # end if - # end for - return (var_vdim, vindex) - -######################################################################## -def local_name_to_diag_name(prop_dict, context=None): -######################################################################## - """ - Translate a local_name to its default diagnostic name. - Currently, this is just equal to the local name. If no local name - exists in the property dictionary, a truncation of the standard - name is used. (256 characters = max length of NetCDF variable name) - >>> local_name_to_diag_name({'local_name':'foo', 'standard_name':'cloud_optical_depth'}) - 'foo' - >>> local_name_to_diag_name({'standard_name':'cloud_optical_depth_layers_from_0p55mu_to_0p99mu'}) - 'cloud_optical_depth_layers_from_0p55mu_to_0p99mu' - >>> local_name_to_diag_name({'units':'km'}) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No standard name or local name to convert to diagnostic name - >>> local_name_to_diag_name({'local_name':'', 'standard_name':'cloud_optical_depth'}) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No standard name or local name to convert to diagnostic name - >>> local_name_to_diag_name({'standard_name':''}) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No standard name or local name to convert to diagnostic name - """ - diag_name = None - if 'local_name' in prop_dict: - locname = prop_dict['local_name'] - if locname: - diag_name = prop_dict['local_name'] - # end if - elif 'standard_name' in prop_dict: - stdname = prop_dict['standard_name'] - if stdname: - maxlen = 256 - diag_name = stdname[:maxlen] - # end if - # end if - - if not diag_name: - emsg = 'No standard name or local name to convert to diagnostic name' - raise CCPPError(emsg) - # end if - - return diag_name - -######################################################################## -def standard_name_to_long_name(prop_dict, context=None): -######################################################################## - """Translate a standard_name to its default long_name - >>> standard_name_to_long_name({'standard_name':'cloud_optical_depth_layers_from_0p55mu_to_0p99mu'}) - 'Cloud optical depth layers from 0.55mu to 0.99mu' - >>> standard_name_to_long_name({'local_name':'foo'}) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No standard name to convert foo to long name - >>> standard_name_to_long_name({}) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No standard name to convert to long name - >>> standard_name_to_long_name({'local_name':'foo'}, context=ParseContext(linenum=3, filename='foo.F90')) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No standard name to convert foo to long name, at foo.F90:4 - >>> standard_name_to_long_name({}, context=ParseContext(linenum=3, filename='foo.F90')) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No standard name to convert to long name, at foo.F90:4 - """ - # We assume that standard_name has been checked for validity - # Make the first char uppercase and replace each underscore with a space - if 'standard_name' in prop_dict: - standard_name = prop_dict['standard_name'] - if standard_name: - long_name = standard_name[0].upper() + re.sub("_", " ", - standard_name[1:]) - else: - long_name = '' - # end if - # Next, substitute a decimal point for the p in [:digit]p[:digit] - match = _REAL_SUBST_RE.match(long_name) - while match is not None: - long_name = match.group(1) + '.' + match.group(2) - match = _REAL_SUBST_RE.match(long_name) - # end while - else: - long_name = '' - if 'local_name' in prop_dict: - lname = ' {}'.format(prop_dict['local_name']) - else: - lname = '' - # end if - ctxt = context_string(context) - emsg = 'No standard name to convert{} to long name{}' - raise CCPPError(emsg.format(lname, ctxt)) - # end if - return long_name - -######################################################################## -def default_kind_val(prop_dict, context=None): -######################################################################## - """Choose a default kind based on a variable's type - >>> default_kind_val({'type':'REAL'}) - 'kind_phys' - >>> default_kind_val({'type':'complex'}) - 'kind_phys' - >>> default_kind_val({'type':'double precision'}) - 'kind_phys' - >>> default_kind_val({'type':'integer'}) - '' - >>> default_kind_val({'type':'character'}) - '' - >>> default_kind_val({'type':'logical'}) - '' - >>> default_kind_val({'local_name':'foo'}) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No type to find default kind for foo - >>> default_kind_val({}) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No type to find default kind - >>> default_kind_val({'local_name':'foo'}, context=ParseContext(linenum=3, filename='foo.F90')) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No type to find default kind for foo, at foo.F90:4 - >>> default_kind_val({}, context=ParseContext(linenum=3, filename='foo.F90')) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No type to find default kind, at foo.F90:4 - """ - if 'type' in prop_dict: - vtype = prop_dict['type'].lower() - if vtype == 'real': - kind = 'kind_phys' - elif vtype == 'complex': - kind = 'kind_phys' - elif FORTRAN_DP_RE.match(vtype) is not None: - kind = 'kind_phys' - else: - kind = '' - # end if - else: - kind = '' - if 'local_name' in prop_dict: - lname = ' {}'.format(prop_dict['local_name']) - errmsg = 'No type to find default kind for {ln}{ct}' - else: - lname = '' - errmsg = 'No type to find default kind{ct}' - # end if - ctxt = context_string(context) - raise CCPPError(errmsg.format(ln=lname, ct=ctxt)) - # end if - return kind - -######################################################################## - -class DimTransform: - """Class to represent a transformation between two variables with - compatible dimensions. - Compatible differences include permutations, sub-selection of the - horizontal dimension, and the ordering of the vertical dimension. - - The "forward" transformation transforms "var1" into "var2" - (i.e., var2 = forward_transform(var1)). - The "reverse" transformation transforms "var2" into "var1" - (i.e., var1 = reverse_transform(var2)). - """ - - def __init__(self, forward_permutation, reverse_permutation, - forward_hdim, forward_hdim_index, forward_vdim_index, - reverse_hdim, reverse_hdim_index, reverse_vdim_index): - """Initialize a dimension transform object. - : A tuple of integers with the location of the - "var1" index for each "var2" index. That is, the first index - for "var2" on the LHS of the forward transform is - [0]. - : A tuple of integers with the location of the - "var2" index for each "var1" index. That is, the first index - for "var1" on the LHS of the forward transform is - [0]. - : The name of the horizontal dimension for "var1". - This is used to determine if an offset needs to be added to - the forward and reverse transforms. - : This is the position of the horizontal dimension - for "var1". For instance, zero means that the horizontal axis is - the fastest varying. - : This is the position of the vertical dimension - for "var1". For instance, zero means that the vertical axis is - the fastest varying. - : The name of the horizontal dimension for "var2". - This is used to determine if an offset needs to be added to - the forward and reverse transforms. - : This is the position of the horizontal dimension - for "var2". For instance, zero means that the horizontal axis is - the fastest varying. - : This is the position of the vertical dimension - for "var2". For instance, zero means that the vertical axis is - the fastest varying. - - # Test that bad inputs are trapped: - >>> DimTransform((0, 1, 2), (2, 1), 'horizontal_dimension', 0, 1, \ - 'horizontal_dimension', \ - 1, 0) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.ParseInternalError: Permutation mismatch, '(0, 1, 2)' and '(2, 1)' - >>> DimTransform((2, 0, 1), (1, 2, 0), 'horizontal_dimension', 3, 2, \ - 'horizontal_dimension', \ - 4, 3) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.ParseInternalError: forward_hdim_index (3) out of range [0, 2] - >>> DimTransform((2, 0, 1), (1, 2, 0), 'horizontal_dimension', 0, 4, \ - 'horizontal_dimension', \ - 4, 3) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.ParseInternalError: forward_vdim_index (4) out of range [0, 2] - >>> DimTransform((2, 0, 1), (1, 2, 0), 'horizontal_dimension', 0, 2, \ - 'horizontal_dimension', \ - 4, 3) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.ParseInternalError: reverse_hdim_index (4) out of range [0, 2] - >>> DimTransform((2, 0, 1), (1, 2, 0), 'horizontal_dimension', 1, 2, \ - 'horizontal_dimension', \ - 0, 3) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.ParseInternalError: reverse_vdim_index (3) out of range [0, 2] - """ - # Store inputs - if len(forward_permutation) != len(reverse_permutation): - emsg = "Permutation mismatch, '{}' and '{}'" - raise ParseInternalError(emsg.format(forward_permutation, - reverse_permutation)) - # end if - self.__forward_perm = forward_permutation - self.__reverse_perm = reverse_permutation - if ((forward_hdim_index < 0) or - (forward_hdim_index >= len(forward_permutation))): - emsg = "forward_hdim_index ({}) out of range [0, {}]" - raise ParseInternalError(emsg.format(forward_hdim_index, - len(forward_permutation)-1)) - # end if - self.__forward_hdim_index = forward_hdim_index - # We cannot test for negative forward_vdim_index because there may - # not be a vertical dimension - if forward_vdim_index >= len(forward_permutation): - emsg = "forward_vdim_index ({}) out of range [0, {}]" - raise ParseInternalError(emsg.format(forward_vdim_index, - len(forward_permutation)-1)) - # end if - self.__forward_vdim_index = forward_vdim_index - if ((reverse_hdim_index < 0) or - (reverse_hdim_index >= len(reverse_permutation))): - emsg = "reverse_hdim_index ({}) out of range [0, {}]" - raise ParseInternalError(emsg.format(reverse_hdim_index, - len(reverse_permutation)-1)) - # end if - self.__reverse_hdim_index = reverse_hdim_index - # We cannot test for negative reverse_vdim_index because there may - # not be a vertical dimension - if reverse_vdim_index >= len(reverse_permutation): - emsg = "reverse_vdim_index ({}) out of range [0, {}]" - raise ParseInternalError(emsg.format(reverse_vdim_index, - len(reverse_permutation)-1)) - # end if - self.__reverse_vdim_index = reverse_vdim_index - # Categorize horizontal dimensions - # v_hloop is True if "var" has extent "horizontal_loop_extent". - # The loop for these variables begins at one while variables with - # extent, "horizontal_dimension" begin at "horizontal_loop_begin" - # during the run phase. - self.__v1_hloop = self.__is_horizontal_loop_dimension(forward_hdim) - if ((not self.__v1_hloop) and - (not ("horizontal_dimension" in forward_hdim))): - emsg = "Uncategorized forward horizontal dimension, '{}'" - raise ParseInternalError(emsg.format(forward_hdim)) - # end if - self.__v2_hloop = self.__is_horizontal_loop_dimension(reverse_hdim) - if ((not self.__v2_hloop) and - (not ("horizontal_dimension" in reverse_hdim))): - emsg = "Uncategorized reverse horizontal dimension, '{}'" - raise ParseInternalError(emsg.format(reverse_hdim)) - # end if - - def forward_transform(self, var2_lname, indices, - adjust_hdim=None, flip_vdim=None): - """Compute and return the LHS of the forward transform from "var1" to - "var2". - is the local name of "var2". - is a tuple of the loop indices for "var1" (i.e., "var1" - will show up in the RHS of the transform as "var1(indices)". - If is not None, it should be a string containing the - local name of the "horizontal_loop_begin" variable. This is used to - compute the offset in the horizontal axis index between one and - "horizontal_loop_begin" (if any). This occurs when one of the - variables has extent "horizontal_loop_extent" and the other has - extent "horizontal_dimension". - If flip_vdim is not None, it should be a string containing the local - name of the vertical extent of the vertical axis for "var1" and - "var2" (i.e., "vertical_layer_dimension" or - "vertical_interface_dimension"). - - # Test forward transform with just horizontal adjustment - >>> DimTransform((0, 1), (0, 1), 'horizontal_dimension', 0, 1, \ - 'horizontal_loop_extent', \ - 0, 1).forward_transform("foo_lhs", ("hind", "vind"), \ - adjust_hdim="col_start") - 'foo_lhs(hind-col_start+1,vind)' - >>> DimTransform((0, 1), (0, 1), 'horizontal_loop_extent', 0, 1, \ - 'horizontal_dimension', \ - 0, 1).forward_transform("foo_lhs", ("hind", "vind"), \ - adjust_hdim="col_start") - 'foo_lhs(hind+col_start-1,vind)' - - # Test flipping vertical dimension - >>> DimTransform((0, 1), (0, 1), 'horizontal_dimension', 0, 1, \ - 'horizontal_dimension', \ - 0, 1).forward_transform("foo_lhs", ("hind", "vind"), \ - flip_vdim="pver") - 'foo_lhs(hind,pver-vind+1)' - - # Test simple permutations - >>> DimTransform((1, 0), (1, 0), 'horizontal_dimension', 0, 1, \ - 'horizontal_dimension', \ - 1, 0).forward_transform("foo_lhs", ("hind", "vind")) - 'foo_lhs(vind,hind)' - >>> DimTransform((2, 0, 1), (1, 2, 0), 'horizontal_dimension', 0, 2, \ - 'horizontal_dimension', \ - 0, 1).forward_transform("foo_lhs", \ - ("hind", "xdim", "vind")) - 'foo_lhs(vind,hind,xdim)' - """ - v2_indices = [indices[x] for x in self.__forward_perm] - if adjust_hdim is not None: - if self.__v1_hloop and (not self.__v2_hloop): - hdim = v2_indices[self.__forward_hdim_index] - adj_str = f"{hdim}+{adjust_hdim}-1" - v2_indices[self.__forward_hdim_index] = adj_str - elif self.__v2_hloop and (not self.__v1_hloop): - hdim = v2_indices[self.__forward_hdim_index] - adj_str = f"{hdim}-{adjust_hdim}+1" - v2_indices[self.__forward_hdim_index] = adj_str - # end if - # end if - if flip_vdim is not None: - vdim = v2_indices[self.__forward_vdim_index] - adj_str = f"{flip_vdim}-{vdim}+1" - v2_indices[self.__forward_vdim_index] = adj_str - # end if - return f"{var2_lname}({','.join(v2_indices)})" - - def reverse_transform(self, var1_lname, indices, - adjust_hdim=None, flip_vdim=None): - """Compute and return the LHS of the forward transform from "var2" to - "var1". - is the local name of "var1". - is a tuple of the loop indices for "var2" (i.e., "var2" - will show up in the RHS of the transform as "var2(indices)". - If is not None, it should be a string containing the - local name of the "horizontal_loop_begin" variable. This is used to - compute the offset in the horizontal axis index between one and - "horizontal_loop_begin" (if any). This occurs when one of the - variables has extent "horizontal_loop_extent" and the other has - extent "horizontal_dimension". - If flip_vdim is not None, it should be a string containing the local - name of the vertical extent of the vertical axis for "var2" and - "var1" (i.e., "vertical_layer_dimension" or - "vertical_interface_dimension"). - - # Test reverse transform with just horizontal adjustment - >>> DimTransform((0, 1), (0, 1), 'horizontal_dimension', 0, 1, \ - 'horizontal_loop_extent', \ - 0, 1).reverse_transform("bar_lhs", ("hind", "vind"), \ - adjust_hdim="col_start") - 'bar_lhs(hind+col_start-1,vind)' - >>> DimTransform((0, 1), (0, 1), 'horizontal_loop_extent', 0, 1, \ - 'horizontal_dimension', \ - 0, 1).reverse_transform("bar_lhs", ("hind", "vind"), \ - adjust_hdim="col_start") - 'bar_lhs(hind-col_start+1,vind)' - - # Test flipping vertical dimension - >>> DimTransform((0, 1), (0, 1), 'horizontal_dimension', 0, 1, \ - 'horizontal_dimension', \ - 0, 1).reverse_transform("bar_lhs", ("hind", "vind"), \ - flip_vdim="pver") - 'bar_lhs(hind,pver-vind+1)' - - # Test simple permutations - >>> DimTransform((1, 0), (1, 0), 'horizontal_dimension', 0, 1, \ - 'horizontal_dimension', \ - 1, 0).reverse_transform("bar_lhs", ("hind", "vind")) - 'bar_lhs(vind,hind)' - >>> DimTransform((2, 0, 1), (1, 2, 0), 'horizontal_dimension', 0, 2, \ - 'horizontal_dimension', \ - 0, 1).reverse_transform("bar_lhs", \ - ("vind", "hind", "xdim")) - 'bar_lhs(hind,xdim,vind)' - """ - v1_indices = [indices[x] for x in self.__reverse_perm] - if adjust_hdim is not None: - if self.__v1_hloop and (not self.__v2_hloop): - hdim = v1_indices[self.__reverse_hdim_index] - adj_str = f"{hdim}-{adjust_hdim}+1" - v1_indices[self.__reverse_hdim_index] = adj_str - elif self.__v2_hloop and (not self.__v1_hloop): - hdim = v1_indices[self.__reverse_hdim_index] - adj_str = f"{hdim}+{adjust_hdim}-1" - v1_indices[self.__reverse_hdim_index] = adj_str - # end if - # end if - if flip_vdim is not None: - vdim = v1_indices[self.__reverse_vdim_index] - adj_str = f"{flip_vdim}-{vdim}+1" - v1_indices[self.__reverse_vdim_index] = adj_str - # end if - return f"{var1_lname}({','.join(v1_indices)})" - - @staticmethod - def __is_horizontal_loop_dimension(hdim): - """Return True if is a run-phase horizontal dimension""" - return (is_horizontal_dimension(hdim) and - ("horizontal_dimension" not in hdim)) - -######################################################################## - -class VariableProperty: - """Class to represent a single property of a metadata header entry - >>> VariableProperty('local_name', str) #doctest: +ELLIPSIS - - >>> VariableProperty('standard_name', str) #doctest: +ELLIPSIS - - >>> VariableProperty('long_name', str) #doctest: +ELLIPSIS - - >>> VariableProperty('units', str) #doctest: +ELLIPSIS - - >>> VariableProperty('dimensions', list) #doctest: +ELLIPSIS - - >>> VariableProperty('type', str) #doctest: +ELLIPSIS - - >>> VariableProperty('kind', str) #doctest: +ELLIPSIS - - >>> VariableProperty('state_variable', str, valid_values_in=['True', 'False', '.true.', '.false.' ], optional_in=True, default_in=False) #doctest: +ELLIPSIS - - >>> VariableProperty('intent', str, valid_values_in=['in', 'out', 'inout']) #doctest: +ELLIPSIS - - >>> VariableProperty('optional', str, valid_values_in=['True', 'False', '.true.', '.false.' ], optional_in=True, default_in=False) #doctest: +ELLIPSIS - - >>> VariableProperty('local_name', str).name - 'local_name' - >>> VariableProperty('standard_name', str).ptype == str - True - >>> VariableProperty('units', str).is_match('units') - True - >>> VariableProperty('units', str).is_match('UNITS') - True - >>> VariableProperty('units', str).is_match('type') - False - >>> VariableProperty('value', int, valid_values_in=[1, 2 ]).valid_value('2') - 2 - >>> VariableProperty('value', int, valid_values_in=[1, 2 ]).valid_value('3') - - >>> VariableProperty('value', int, valid_values_in=[1, 2 ]).valid_value('3', error=True) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: Invalid value variable property, '3' - >>> VariableProperty('units', str, check_fn_in=check_units).valid_value('m s-1') - 'm s-1' - >>> VariableProperty('units', str, check_fn_in=check_units).valid_value(' ') - - >>> VariableProperty('units', str, check_fn_in=check_units).valid_value(' ', error=True) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: ' ' is not a valid unit - >>> VariableProperty('dimensions', list, check_fn_in=check_dimensions).valid_value('()') - [] - >>> VariableProperty('dimensions', list, check_fn_in=check_dimensions).valid_value('(x)') - ['x'] - >>> VariableProperty('dimensions', list, check_fn_in=check_dimensions).valid_value('x') - - >>> VariableProperty('dimensions', list, check_fn_in=check_dimensions).valid_value('(x:y)') - ['x:y'] - >>> VariableProperty('dimensions', list, check_fn_in=check_dimensions).valid_value('(w:x,y:z)') - ['w:x', 'y:z'] - >>> VariableProperty('dimensions', list, check_fn_in=check_dimensions).valid_value(['size(foo)']) - ['size(foo)'] - >>> VariableProperty('dimensions', list, check_fn_in=check_dimensions).valid_value('(w:x,x:y:z:q)', error=True) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: 'x:y:z:q' is an invalid dimension range - >>> VariableProperty('dimensions', list, check_fn_in=check_dimensions).valid_value('(x:3y)', error=True) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: '3y' is not a valid Fortran identifier - >>> VariableProperty('local_name', str, check_fn_in=check_local_name).valid_value('foo') - 'foo' - >>> VariableProperty('local_name', str, check_fn_in=check_local_name).valid_value('foo(bar)') - 'foo(bar)' - >>> VariableProperty('local_name', str, check_fn_in=check_local_name).valid_value('q(:,:,index_of_water_vapor_specific_humidity)') - 'q(:,:,index_of_water_vapor_specific_humidity)' - >>> VariableProperty('molar_mass', float, check_fn_in=check_molar_mass).valid_value('12.1') - 12.1 - """ - - __true_vals = ['t', 'true', '.true.'] - __false_vals = ['f', 'false', '.false.'] - - def __init__(self, name_in, type_in, valid_values_in=None, - optional_in=False, default_in=None, default_fn_in=None, - check_fn_in=None, mult_entry_ok=False): - """Conduct sanity checks and initialize this variable property.""" - self._name = name_in - self._type = type_in - if self._type not in [bool, int, list, str, float]: - emsg = "{} has invalid VariableProperty type, '{}'" - raise CCPPError(emsg.format(name_in, type_in)) - # end if - self._valid_values = valid_values_in - self._optional = optional_in - self._default = None - self._default_fn = None - if self.optional: - if (default_in is None) and (default_fn_in is None): - emsg = 'default_in or default_fn_in is a required property for {} because it is optional' - raise CCPPError(emsg.format(name_in)) - if (default_in is not None) and (default_fn_in is not None): - emsg = 'default_in and default_fn_in cannot both be provided' - raise CCPPError(emsg) - self._default = default_in - self._default_fn = default_fn_in - elif default_in is not None: - emsg = 'default_in is not a valid property for {} because it is not optional' - raise CCPPError(emsg.format(name_in)) - elif default_in is not None: - emsg = 'default_fn_in is not a valid property for {} because it is not optional' - raise CCPPError(emsg.format(name_in)) - self._check_fn = check_fn_in - self._add_multiple_ok = mult_entry_ok - - @property - def name(self): - """Return the name of the property""" - return self._name - - @property - def ptype(self): - """Return the type of the property""" - return self._type - - @property - def has_default_func(self): - """Return True iff this variable property has a default function""" - return self._default_fn is not None - - def get_default_val(self, prop_dict, context=None): - """Return this variable property's default value or raise an - exception if there is no default value or default value function.""" - if self.has_default_func: - return self._default_fn(prop_dict, context) - # end if - if self._default is not None: - return self._default - # end if - ctxt = context_string(context) - emsg = 'No default for variable property {}{}' - raise CCPPError(emsg.format(self.name, ctxt)) - - - @property - def optional(self): - """Return True iff this variable property is optional""" - return self._optional - - @property - def add_multiple(self): - """Return True iff multiple entries of this property should be - accumulated. If False, it should either be an error or new - instances should replace the old, however, this functionality - must be implemented by the calling routine (e.g., Var)""" - return self._add_multiple_ok - - def is_match(self, test_name): - """Return True iff is the name of this property""" - return self.name.lower() == test_name.lower() - - def valid_value(self, test_value, prop_dict=None, error=False): - """Return a valid version of if it is valid. - If is not valid, return None or raise an exception, - depending on the value of . - If is not None, it may be used in value validation. - """ - valid_val = None - if self.ptype is int: - try: - tval = int(test_value) - if self._valid_values is not None: - if tval in self._valid_values: - valid_val = tval - else: - valid_val = None # i.e. pass - else: - valid_val = tval - except CCPPError: - valid_val = None # Redundant but more expressive than pass - elif self.ptype is float: - try: - tval = float(test_value) - if self._valid_values is not None: - if tval in self._valid_values: - valid_val = tval - else: - valid_val = None # i.e. pass - # end if - else: - valid_val = tval - # end if - except CCPPError: - valid_val = None - # end try - elif self.ptype is list: - if isinstance(test_value, str): - tval = fortran_list_match(test_value) - if tval and (len(tval) == 1) and (not tval[0]): - # Scalar - tval = list() - # end if - else: - tval = test_value - # end if - if isinstance(tval, list): - valid_val = tval - elif isinstance(tval, tuple): - valid_val = list(tval) - else: - valid_val = None - # end if - if (valid_val is not None) and (self._valid_values is not None): - # Special case for lists, _valid_values applies to elements - for item in valid_val: - if item not in self._valid_values: - valid_val = None - break - # end if - # end for - else: - pass - elif self.ptype is bool: - if isinstance(test_value, str): - if test_value.lower() in VariableProperty.__true_vals + VariableProperty.__false_vals: - valid_val = test_value.lower() in VariableProperty.__true_vals - else: - valid_val = None # i.e., pass - # end if - else: - valid_val = not not test_value # pylint: disable=unneeded-not - elif self.ptype is str: - if isinstance(test_value, str): - if self._valid_values is not None: - if test_value in self._valid_values: - valid_val = test_value - else: - valid_val = None # i.e., pass - else: - valid_val = test_value - # end if - # end if - # end if - # Call a check function? - if valid_val and (self._check_fn is not None): - valid_val = self._check_fn(valid_val, prop_dict, error) - elif (valid_val is None) and error: - emsg = "Invalid {} variable property, '{}'" - raise CCPPError(emsg.format(self.name, test_value)) - # end if - return valid_val - -############################################################################## - -class VarCompatObj: - """Class to compare two Var objects and then answer questions about - the compatibility of the two variables. - There are three levels of compatibility. - * Compatible is when two variables match in all properties so that one - can be passed to another with no transformation. - * Comformable is when two variables have the same information but may - need some transformation between them. Examples are differences in - dimension ordering, units, or kind. - * Not Compatible is when information from one variable cannot be passed - to the other. - - Note that character(len=*) is considered equivalent to - character(len=) - - # Test that we can create a standard VarCompatObj object - >>> from parse_tools import init_log, set_log_to_null - >>> _DOCTEST_LOGGING = init_log('var_props') - >>> set_log_to_null(_DOCTEST_LOGGING) - >>> _DOCTEST_RUNENV = CCPPFrameworkEnv(_DOCTEST_LOGGING, \ - ndict={'host_files':'', \ - 'scheme_files':'', \ - 'suites':''}, \ - kind_types=["kind_phys=REAL64", \ - "kind_dyn=REAL32", \ - "kind_host=REAL64"]) - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", [], "var1_lname", False,\ - "var_stdname", "real", "kind_phys", "m", [], "var2_lname", False,\ - _DOCTEST_RUNENV) #doctest: +ELLIPSIS - - - # Test that a 2-D var with no horizontal transform works - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", ['horizontal_dimension'], "var1_lname", False, \ - "var_stdname", "real", "kind_phys", "m", ['horizontal_dimension'], "var2_lname", False, \ - _DOCTEST_RUNENV) #doctest: +ELLIPSIS - - - # Test that a 2-D var with a horizontal transform works - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", ['horizontal_dimension'], "var1_lname", False, \ - "var_stdname", "real", "kind_phys", "m", ['horizontal_loop_extent'], "var2_lname", False, \ - _DOCTEST_RUNENV) #doctest: +ELLIPSIS - - - # Test that a 1-D var with no vertical transform works - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", ['vertical_layer_dimension'], "var1_lname", False, \ - "var_stdname", "real", "kind_phys", "m", ['vertical_layer_dimension'], "var2_lname", False, \ - _DOCTEST_RUNENV) #doctest: +ELLIPSIS - - - # Test that a 1-D var with vertical flipping works and that it - # produces the correct reverse transformation - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", ['vertical_layer_dimension'], "var1_lname", False,\ - "var_stdname", "real", "kind_phys", "m", ['vertical_layer_dimension'], "var2_lname", True, \ - _DOCTEST_RUNENV).reverse_transform("var1_lname", "var2_lname", ('k',), ('nk-k+1',)) - 'var1_lname(nk-k+1) = var2_lname(k)' - - # Test that unit conversions with a scalar var works - >>> VarCompatObj("var_stdname", "real", "kind_phys", "Pa", [], "var1_lname", False, \ - "var_stdname", "real", "kind_phys", "hPa", [], "var2_lname", False, \ - _DOCTEST_RUNENV).forward_transform("var1_lname", "var2_lname", [], []) #doctest: +ELLIPSIS - 'var1_lname = 1.0E-2_kind_phys*var2_lname' - - # Test that unit conversions with a scalar var works - >>> VarCompatObj("var_stdname", "real", "kind_phys", "Pa", [], "var1_lname", False, \ - "var_stdname", "real", "kind_phys", "hPa", [], "var2_lname", False, \ - _DOCTEST_RUNENV).reverse_transform("var1_lname", "var2_lname", [], []) #doctest: +ELLIPSIS - 'var1_lname = 1.0E+2_kind_phys*var2_lname' - - # Test that a 2-D var with unit conversion m->km works - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", ['horizontal_dimension'], "var1_lname", False, \ - "var_stdname", "real", "kind_phys", "km", ['horizontal_dimension'], "var2_lname", False, \ - _DOCTEST_RUNENV) #doctest: +ELLIPSIS - - - # Test that a 2-D var with unit conversion m->km works and that it - # produces the correct forward transformation - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", ['horizontal_dimension'], "var1_lname", False, \ - "var_stdname", "real", "kind_phys", "km", ['horizontal_dimension'], "var2_lname", False, \ - _DOCTEST_RUNENV).forward_transform("var1_lname", "var2_lname", 'i', 'i') - 'var1_lname(i) = 1.0E-3_kind_phys*var2_lname(i)' - - # Test that a 3-D var with unit conversion m->km and vertical flipping - # works and that it produces the correct reverse transformation - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", ['horizontal_dimension', 'vertical_layer_dimension'], "var1_lname", False,\ - "var_stdname", "real", "kind_phys", "km",['horizontal_dimension', 'vertical_layer_dimension'], "var2_lname", True, \ - _DOCTEST_RUNENV).reverse_transform("var1_lname", "var2_lname", ('i','k'), ('i','nk-k+1')) - 'var1_lname(i,nk-k+1) = 1.0E+3_kind_phys*var2_lname(i,k)' - - # Test that a 2-D var with equivalent units works and that it - # skips any unit transformations - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m2 s-2", ['horizontal_dimension'], "var1_lname", False, \ - "var_stdname", "real", "kind_phys", "J kg-1", ['horizontal_dimension'], "var2_lname", False, \ - _DOCTEST_RUNENV).forward_transform("var1_lname", "var2_lname", 'i', 'i') - 'var1_lname(i) = var2_lname(i)' - - # Test that a 2-D var with identical units works and that it - # skips any unit transformations - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m2 s-2", ['horizontal_dimension'], "var1_lname", False, \ - "var_stdname", "real", "kind_phys", "m+2 s-2", ['horizontal_dimension'], "var2_lname", False, \ - _DOCTEST_RUNENV).forward_transform("var1_lname", "var2_lname", 'i', 'i') - 'var1_lname(i) = var2_lname(i)' - """ - - def __init__(self, var1_stdname, var1_type, var1_kind, var1_units, - var1_dims, var1_lname, var1_top, var2_stdname, var2_type, var2_kind, - var2_units, var2_dims, var2_lname, var2_top, run_env, v1_context=None, - v2_context=None, is_tend=False): - """Initialize this object with information on the equivalence and/or - conformability of two variables. - variable 1 is described by , , , - , , , , and . - variable 2 is described by , , , - , , , , and . - is the CCPPFrameworkEnv object used here to verify kind - equivalence or to produce kind transformations. - is a flag where, if true, we are validating a tendency variable (var1) - against it's equivalent state variable (var2) - """ - self.__equiv = True # No transformation required - self.__compat = True # Callable with transformation - self.__stdname = var1_stdname - self.__v1_context = v1_context - self.__v2_context = v2_context - self.__v1_kind = var1_kind - self.__v2_kind = var2_kind - self.v1_units = var1_units - self.v2_units = var2_units - self.v1_stdname = var1_stdname - self.v2_stdname = var2_stdname - # Default (null) transform information - self.__dim_transforms = None - self.__kind_transforms = None - self.__unit_transforms = None - self.has_vert_transforms = False - incompat_reason = list() - # First, check for fatal incompatibilities - # If it's a tendency variable, the standard name should be of the - # form "tendency_of_var2_stdname" - if is_tend and not var1_stdname.startswith('tendency_of'): - self.__equiv = False - self.__compat = False - incompat_reason.append('not a tendency variable') - if not is_tend and var1_stdname != var2_stdname: - self.__equiv = False - self.__compat = False - incompat_reason.append("standard names") - # end if - if var1_type != var2_type: - self.__equiv = False - self.__compat = False - incompat_reason.append("types") - # end if - # Check kind argument - if self.__compat: - if var1_type == 'character': - # First, make sure we have supported character 'kind' args: - v1_kind = self.char_kind_check(var1_kind) - if not v1_kind: - ctx = context_string(v1_context) - emsg = "Unsupported character kind/len argument, '{}', " - emsg += "in {}{}" - incompat_reason.append(emsg.format(var1_kind, - var1_lname, ctx)) - # end if - self.__v1_kind = None - v2_kind = self.char_kind_check(var2_kind) - if not v2_kind: - ctx = context_string(v2_context) - emsg = "Unsupported character kind/len argument, '{}', " - emsg += "in {}{}" - incompat_reason.append(emsg.format(var2_kind, - var2_lname, ctx)) - # end if - self.__v2_kind = None - # Character types have to 'match' or the variables are - # incompatible - kind_eq = ((v1_kind and v2_kind) and - ((v1_kind == v2_kind) or - (((v1_kind == 'len=*') and - (v2_kind.startswith('len='))) or - (v1_kind.startswith('len=') and - (v2_kind == 'len=*'))))) - if not kind_eq: - self.__equiv = False - self.__compat = False - incompat_reason.append("character len arguments") - # end if - else: - if var1_kind != var2_kind: - self.__kind_transforms = self._get_kind_convstrs(var1_kind, - var2_kind, - run_env) - self.__equiv = self.__kind_transforms is None - # end if - # end if - # end if - if self.__compat: - # Only "none" units are case-insensitive - if var1_units.lower() == 'none': - var1_units = 'none' - # end if - if var2_units.lower() == 'none': - var2_units = 'none' - # end if - # Check units argument - if is_tend: - # A tendency variable's units should be " s-1" - tendency_split_units = var1_units.split('s-1')[0].strip() - if tendency_split_units != var2_units: - # We don't currently support unit conversions for tendency variables, - # but we can check if the units are identical or equivalent - unit_transforms = self._get_unit_convstrs(tendency_split_units, - var2_units) - if not unit_transforms == (None, None): - emsg = f"\nMismatch tendency variable units '{var1_units}'" - emsg += f" for variable '{var1_stdname}'." - emsg += " No variable transforms supported for tendencies." - emsg += f" Tendency units should be '{var2_units} s-1' to match state variable." - self.__equiv = False - self.__compat = False - incompat_reason.append(emsg) - # end if - elif var1_units != var2_units: - # Try to find a set of unit conversions - unit_transforms = self._get_unit_convstrs(var1_units, - var2_units) - # Handle equivalent or identical units = (None, None) - if not unit_transforms == (None, None): - self.__equiv = False - self.__unit_transforms = unit_transforms - # end if - # end if - if self.__compat: - # Check for vertical array flipping (do later) - if var1_top != var2_top: - self.__compat = True - self.has_vert_transforms = True - # end if - # end if - if self.__compat: - # Check dimensions - if var1_dims or var2_dims: - _, vdim_ind = find_vertical_dimension(var1_dims) - if (var1_dims != var2_dims): - self.__dim_transforms = self._get_dim_transforms(var1_dims, - var2_dims) - self.__compat = self.__dim_transforms is not None - # end if - # end if - if not self.__compat: - incompat_reason.append('dimensions') - # end if - # end if - self.__incompat_reason = " and ".join([x for x in incompat_reason if x]) - - def forward_transform(self, lvar_lname, rvar_lname, rvar_indices, lvar_indices, - adjust_hdim=None, flip_vdim=None): - """Compute and return the the forward transform from "var1" to "var2". - is the local name of "var2". - is the local name of "var1". - is a tuple of the loop indices for "var1" (i.e., "var1" - will show up in the RHS of the transform as "var1(rvar_indices)". - is a tuple of the loop indices for "var2" (i.e., "var2" - will show up in the LHS of the transform as "var2(lvar_indices)". - If is not None, it should be a string containing the - local name of the "horizontal_loop_begin" variable. This is used to - compute the offset in the horizontal axis index between one and - "horizontal_loop_begin" (if any). This occurs when one of the - variables has extent "horizontal_loop_extent" and the other has - extent "horizontal_dimension". - If flip_vdim is not None, it should be a string containing the local - name of the vertical extent of the vertical axis for "var1" and - "var2" (i.e., "vertical_layer_dimension" or - "vertical_interface_dimension"). - """ - # Dimension transform (Indices handled externally) - if len(rvar_indices) == 0: - rhs_term = f"{rvar_lname}" - lhs_term = f"{lvar_lname}" - else: - rhs_term = f"{rvar_lname}({','.join(rvar_indices)})" - lhs_term = f"{lvar_lname}({','.join(lvar_indices)})" - # end if - - if self.has_kind_transforms: - kind = self.__kind_transforms[1] - rhs_term = f"real({rhs_term}, {kind})" - else: - kind = '' - # end if - if self.has_unit_transforms: - if kind: - kind = "_" + kind - elif self.__v2_kind: - kind = "_" + self.__v2_kind - # end if - rhs_term = self.__unit_transforms[0].format(var=rhs_term, kind=kind) - # end if - return f"{lhs_term} = {rhs_term}" - - def reverse_transform(self, lvar_lname, rvar_lname, rvar_indices, lvar_indices, - adjust_hdim=None, flip_vdim=None): - """Compute and return the the reverse transform from "var2" to "var1". - is the local name of "var1". - is the local name of "var2". - is a tuple of the loop indices for "var1" (i.e., "var1" - will show up in the RHS of the transform as "var1(rvar_indices)". - is a tuple of the loop indices for "var2" (i.e., "var2" - will show up in the LHS of the transform as "var2(lvar_indices)". - If is not None, it should be a string containing the - local name of the "horizontal_loop_begin" variable. This is used to - compute the offset in the horizontal axis index between one and - "horizontal_loop_begin" (if any). This occurs when one of the - variables has extent "horizontal_loop_extent" and the other has - extent "horizontal_dimension". - If flip_vdim is not None, it should be a string containing the local - name of the vertical extent of the vertical axis for "var1" and - "var2" (i.e., "vertical_layer_dimension" or - "vertical_interface_dimension"). - """ - # Dimension transforms (Indices handled externally) - if len(rvar_indices) == 0: - rhs_term = f"{rvar_lname}" - lhs_term = f"{lvar_lname}" - else: - lhs_term = f"{lvar_lname}({','.join(lvar_indices)})" - rhs_term = f"{rvar_lname}({','.join(rvar_indices)})" - # end if - - if self.has_kind_transforms: - kind = self.__kind_transforms[0] - rhs_term = f"real({rhs_term}, {kind})" - else: - kind = '' - # end if - if self.has_unit_transforms: - if kind: - kind = "_" + kind - elif self.__v1_kind: - kind = "_" + self.__v1_kind - # end if - rhs_term = self.__unit_transforms[1].format(var=rhs_term, kind=kind) - # end if - return f"{lhs_term} = {rhs_term}" - - def _get_kind_convstrs(self, var1_kind, var2_kind, run_env): - """Attempt to determine if no transformation is required (i.e., if - and will be the same at runtime. If so, - return None. - If a conversion is required, return a tuple with the two kinds, - i.e., (var1_kind, var2_kind). - - # Initial setup - >>> from parse_tools import init_log, set_log_to_null - >>> _DOCTEST_LOGGING = init_log('var_props') - >>> set_log_to_null(_DOCTEST_LOGGING) - >>> _DOCTEST_RUNENV = CCPPFrameworkEnv(_DOCTEST_LOGGING, \ - ndict={'host_files':'', \ - 'scheme_files':'', \ - 'suites':''}, \ - kind_types=["kind_phys=REAL64", \ - "kind_dyn=REAL32", \ - "kind_host=REAL64"]) - >>> _DOCTEST_CONTEXT1 = ParseContext(linenum=3, filename='foo.F90') - >>> _DOCTEST_CONTEXT2 = ParseContext(linenum=5, filename='bar.F90') - >>> _DOCTEST_VCOMPAT = VarCompatObj("var_stdname", "real", "kind_phys", \ - "m", [], "var1_lname", False, "var_stdname", \ - "real", "kind_phys", "m", [], \ - "var2_lname", False, _DOCTEST_RUNENV, \ - v1_context=_DOCTEST_CONTEXT1, \ - v2_context=_DOCTEST_CONTEXT2) - - # Try some kind conversions - >>> _DOCTEST_VCOMPAT._get_kind_convstrs('kind_phys', 'kind_dyn', \ - _DOCTEST_RUNENV) - ('kind_phys', 'kind_dyn') - >>> _DOCTEST_VCOMPAT._get_kind_convstrs('kind_phys', 'REAL32', \ - _DOCTEST_RUNENV) - ('kind_phys', 'REAL32') - - # Try some non-conversions - >>> _DOCTEST_VCOMPAT._get_kind_convstrs('kind_phys', 'kind_host', \ - _DOCTEST_RUNENV) - - >>> _DOCTEST_VCOMPAT._get_kind_convstrs('REAL64', 'kind_host', \ - _DOCTEST_RUNENV) - - """ - kind1 = run_env.kind_spec(var1_kind) - if kind1 is None: - kind1 = var1_kind - # end if - kind2 = run_env.kind_spec(var2_kind) - if kind2 is None: - kind2 = var2_kind - # end if - if kind1 != kind2: - return (var1_kind, var2_kind) - # end if - return None - - def _get_unit_convstrs(self, var1_units, var2_units): - """Attempt to retrieve the forward and reverse unit transformations - for transforming a variable in to / from a variable in - . Return (None, None) if units are equivalent or identical - after parsing (this can happen when comparing m2 and m+2). - - # Initial setup - >>> from parse_tools import init_log, set_log_to_null - >>> _DOCTEST_LOGGING = init_log('var_props') - >>> set_log_to_null(_DOCTEST_LOGGING) - >>> _DOCTEST_RUNENV = CCPPFrameworkEnv(_DOCTEST_LOGGING, \ - ndict={'host_files':'', \ - 'scheme_files':'', \ - 'suites':''}, \ - kind_types=["kind_phys=REAL64", \ - "kind_dyn=REAL32", \ - "kind_host=REAL64"]) - >>> _DOCTEST_CONTEXT1 = ParseContext(linenum=3, filename='foo.F90') - >>> _DOCTEST_CONTEXT2 = ParseContext(linenum=5, filename='bar.F90') - >>> _DOCTEST_VCOMPAT = VarCompatObj("var_stdname", "real", "kind_phys", \ - "m", [], "var1_lname", False, "var_stdname", \ - "real", "kind_phys", "m", [], \ - "var2_lname", False, _DOCTEST_RUNENV, \ - v1_context=_DOCTEST_CONTEXT1, \ - v2_context=_DOCTEST_CONTEXT2) - - # Try some working unit transforms - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('m', 'mm') - ('1.0E+3{kind}*{var}', '1.0E-3{kind}*{var}') - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('kg kg-1', 'g kg-1') - ('1.0E+3{kind}*{var}', '1.0E-3{kind}*{var}') - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('C', 'K') - ('{var}+273.15{kind}', '{var}-273.15{kind}') - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('V A', 'W') - (None, None) - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('N m-2', 'Pa') - (None, None) - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('m2 s-2', 'J kg-1') - (None, None) - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('m+2 s-2', 'J kg-1') - (None, None) - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('m+2 s-2', 'm2 s-2') - (None, None) - - # Try an invalid conversion - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('1', 'none') #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.ParseSyntaxError: Unsupported unit conversion, '1' to 'none' for 'var_stdname' - - # Try an unsupported conversion - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('C', 'm') #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.ParseSyntaxError: Unsupported unit conversion, 'C' to 'm' for 'var_stdname' - """ - u1_str = self.units_to_string(var1_units, self.__v1_context) - u2_str = self.units_to_string(var2_units, self.__v2_context) - # If u1_str and u2_str are identical, for example after parsing - # "m2 s-2" and "m+2 s-2", return (None, None) to signal that - # the units are in fact identical - if u1_str == u2_str: - return (None, None) - unit_conv_str = "{0}__to__{1}".format(u1_str, u2_str) - try: - forward_transform = getattr(unit_conversion, unit_conv_str)() - except AttributeError: - emsg = "Unsupported unit conversion, '{}' to '{}' for '{}'" - raise ParseSyntaxError(emsg.format(var1_units, var2_units, - self.__stdname, - context=self.__v2_context)) - # end if - unit_conv_str = "{0}__to__{1}".format(u2_str, u1_str) - try: - reverse_transform = getattr(unit_conversion, unit_conv_str)() - except AttributeError: - emsg = "Unsupported unit conversion, '{}' to '{}' for '{}'" - raise ParseSyntaxError(emsg.format(var2_units, var1_units, - self.__stdname, - context=self.__v1_context)) - # end if - # For equivalent units, return (None, None) - if forward_transform == '{var}' and reverse_transform == '{var}': - return (None, None) - else: - return (forward_transform, reverse_transform) - - def _get_dim_transforms(self, var1_dims, var2_dims): - """Attempt to find forward and reverse permutations for transforming a - variable with shape, , to / from a variable with shape, - . - Return the permutations, or None. - The forward dimension transformation is a permutation of the indices of - the first variable to the second. - The reverse dimension transformation is a permutation of the indices of - the second variable to the first. - - # Initial setup - >>> from parse_tools import init_log, set_log_to_null - >>> _DOCTEST_LOGGING = init_log('var_props') - >>> set_log_to_null(_DOCTEST_LOGGING) - >>> _DOCTEST_RUNENV = CCPPFrameworkEnv(_DOCTEST_LOGGING, \ - ndict={'host_files':'', \ - 'scheme_files':'', \ - 'suites':''}, \ - kind_types=["kind_phys=REAL64", \ - "kind_dyn=REAL32", \ - "kind_host=REAL64"]) - >>> _DOCTEST_CONTEXT1 = ParseContext(linenum=3, filename='foo.F90') - >>> _DOCTEST_CONTEXT2 = ParseContext(linenum=5, filename='bar.F90') - >>> _DOCTEST_VCOMPAT = VarCompatObj("var_stdname", "real", "kind_phys", \ - "m", [], "var1_lname", False, "var_stdname", \ - "real", "kind_phys", "m", [], \ - "var2_lname", False, _DOCTEST_RUNENV, \ - v1_context=_DOCTEST_CONTEXT1, \ - v2_context=_DOCTEST_CONTEXT2) - - # Test simple permutations - >>> _DOCTEST_VCOMPAT._get_dim_transforms(['horizontal_dimension', \ - 'vertical_layer_dimension'], \ - ['vertical_layer_dimension', \ - 'horizontal_dimension']) \ - #doctest: +ELLIPSIS - - >>> _DOCTEST_VCOMPAT._get_dim_transforms(['horizontal_dimension', \ - 'vertical_layer_dimension', \ - 'xdim'], \ - ['vertical_layer_dimension', \ - 'horizontal_dimension', \ - 'xdim']) #doctest: +ELLIPSIS - - >>> _DOCTEST_VCOMPAT._get_dim_transforms(['horizontal_dimension', \ - 'vertical_layer_dimension', \ - 'xdim'], \ - ['xdim', \ - 'horizontal_dimension', \ - 'vertical_layer_dimension']) \ - #doctest: +ELLIPSIS - - - # Test some mismatch sets - >>> _DOCTEST_VCOMPAT._get_dim_transforms(['horizontal_dimension', \ - 'vertical_layer_dimension', \ - 'xdim'], \ - ['horizontal_dimension', \ - 'vertical_layer_dimension']) \ - - >>> _DOCTEST_VCOMPAT._get_dim_transforms(['horizontal_dimension', \ - 'vertical_layer_dimension', \ - 'xdim'], \ - ['horizontal_dimension', \ - 'vertical_layer_dimension', \ - 'ydim']) - - """ - transforms = None - v1_dims = self.__regularize_dimensions(var1_dims) - v2_dims = self.__regularize_dimensions(var2_dims) - if v1_dims != v2_dims: - self.__equiv = False - # end if - # Is v2 a permutation of v1? - if len(v1_dims) == len(v2_dims): - v1_set = sorted(v1_dims) - v2_set = sorted(v2_dims) - if v1_set == v2_set: - forward_permutation = list() - reverse_permutation = [None] * len(v1_dims) - forward_hdim = '' - forward_hdim_index = -1 - forward_vdim_index = -1 - reverse_hdim = '' - reverse_hdim_index = -1 - reverse_vdim_index = -1 - for v2index, v2dim in enumerate(v2_dims): - for v1index, v1dim in enumerate(v1_dims): - if v1dim == v2dim: - # Add check for repeated indices - if v1index not in forward_permutation: - forward_permutation.append(v1index) - reverse_permutation[v1index] = v2index - if is_horizontal_dimension(var1_dims[v1index]): - forward_hdim = var1_dims[v1index] - forward_hdim_index = v1index - reverse_hdim = var2_dims[v2index] - reverse_hdim_index = v2index - elif is_vertical_dimension(var1_dims[v1index]): - forward_vdim_index = v1index - reverse_vdim_index = v2index - # end if - break - # end if - # end if (hope there is a repeated dimension) - # end for - # end for - if len(forward_permutation) != len(v1_dims): - emsg = "Bad dimension handling, '{}' and '{}'" - raise ParseInternalError(emsg.format(var1_dims, var2_dims)) - # end if - transforms = DimTransform(forward_permutation, - reverse_permutation, - forward_hdim, forward_hdim_index, - forward_vdim_index, - reverse_hdim, reverse_hdim_index, - reverse_vdim_index) - # end if - # end if - return transforms - - @staticmethod - def char_kind_check(kind_str): - """If is a supported character 'kind' argument, return its - standardized form, otherwise return False. - """ - kind_ok = False - if isinstance(kind_str, str): - # Character allows both len and kind but we only support len - kentries = [x.strip() for x in kind_str.split(',') if x.strip()] - if len(kentries) == 1: - if kentries[0][0:4].lower() == 'len=': - kind_ok = True - # end if (no else, kind_ok already False) - # end if (no else, kind_ok already False) - # end if (no else, kind_ok already False) - return kind_ok - - def units_to_string(self, units, context=None): - """Replace variable unit description with string that is a legal - Python identifier. - If the resulting string is a Python keyword, raise an exception.""" - # Start with breaking up the string by spaces - items = units.split() - # Identify units with positive exponents - # without a plus sign (m2 instead of m+2). - pattern = re.compile(r"([a-zA-Z]+)([0-9]+)") - for index, item in enumerate(items): - match = pattern.match(item) - if match: - items[index] = "+".join(match.groups()) - # Combine list into string using underscores - string = "_".join(items) - # Replace each minus sign with '_minus_' - string = string.replace("-","_minus_") - # Replace each plus sign with '_plus_' - string = string.replace("+","_plus_") - # "1" is a valid unit - if string == "1": - string = "one" - # end if - # Test that the resulting string is a valid Python identifier - if not string.isidentifier(): - emsg = "Unsupported units entry for {}, '{}'{}" - ctx = context_string(context) - raise ParseSyntaxError(emsg.format(self.__stdname, units ,ctx)) - # end if - # Test that the resulting string is NOT a Python keyword - if keyword.iskeyword(string): - emsg = "Invalid units entry, '{}', Python identifier" - raise ParseSyntaxError(emsg.format(units), - context=context) - # end if - return string - - @staticmethod - def __regularize_dimensions(dims): - """Regularize by substituting a standin for any horizontal - dimension description (e.g., 'ccpp_constant_one:horizontal_loop_extent', - 'horizontal_loop_begin:horizontal_loop_end'). Also, regularize all - other dimensions by adding 'ccpp_constant_one' to any singleton - dimension. - Return the regularized dimensions. - """ - new_dims = list() - for dim in dims: - if is_horizontal_dimension(dim): - new_dims.append(_HDIM_TEMPNAME) - elif ':' not in dim: - new_dims.append('ccpp_constant_one:' + dim) - else: - new_dims.append(dim) - # end if - # end for - return new_dims - - @property - def incompat_reason(self): - """Return the reason(s) the two variables are incompatible (or an - empty string)""" - return self.__incompat_reason - - @property - def equiv(self): - """Return True if this object describes two Var objects which are - equivalent (i.e., no transformation required to pass one to the other). - """ - return self.__equiv - - @property - def compat(self): - """Return True if this object describes two Var objects which are - compatible (i.e., the values from one can be transferred to the other - via the transformation(s) described in the object). - """ - return self.__compat - - @property - def has_dim_transforms(self): - """Return True if this object has dimension transformations. - The dimension transformations is a tuple for forward and reverse - transformation. - The forward dimension transformation is a permutation of the indices of - the first variable to the second. - The reverse dimension transformation is a permutation of the indices of - the second variable to the first. - """ - return self.__dim_transforms is not None - - @property - def has_kind_transforms(self): - """Return True if this object has the kind transformation. - The kind transformation is a tuple containing the forward and reverse - kind transformations. - The forward kind transformation is a string representation of the - kind of the second variable. - The reverse kind transformation is a string representation of the - kind of the first variable. - """ - return self.__kind_transforms is not None - - @property - def has_unit_transforms(self): - """Return True if this object has the unit transformations. - The unit transformations is a tuple with forward and reverse unit - transformations. - The forward unit transformation is a string representation of the - equation to transform the first variable into the units of the second - The reverse unit transformation is a string representation of the - equation to transform the second variable into the units of the first - Each unit transform is a string which can be formatted with - and arguments to produce code to transform one variable into - the correct units of the other. - """ - return self.__unit_transforms is not None and self.__unit_transforms[0] - - def __bool__(self): - """Return True if this object describes two Var objects which are - equivalent (i.e., no transformation required to pass one to the other). - """ - return self.equiv - -############################################################################### diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt deleted file mode 100644 index a7428a77..00000000 --- a/src/CMakeLists.txt +++ /dev/null @@ -1,45 +0,0 @@ -include(GNUInstallDirs) - -#------------------------------------------------------------------------------ -# Set the sources -set(SOURCES_F90 - ccpp_types.F90 -) - -set(CMAKE_Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_INCLUDEDIR}) - -#------------------------------------------------------------------------------ -# Define the executable and what to link -add_library(ccpp_framework STATIC ${SOURCES_F90}) -target_link_libraries(ccpp_framework PUBLIC MPI::MPI_Fortran) -if(OPENMP) - target_link_libraries(ccpp_framework PUBLIC OpenMP::OpenMP_Fortran) -endif() -set_target_properties(ccpp_framework PROPERTIES - VERSION ${PROJECT_VERSION} - SOVERSION ${PROJECT_VERSION_MAJOR}) - -#------------------------------------------------------------------------------ -# Installation -# -target_include_directories(ccpp_framework PUBLIC - INTERFACE $ - $ -) - - -# Define where to install the library -install(TARGETS ccpp_framework - EXPORT ccpp_framework-targets - ARCHIVE DESTINATION lib - LIBRARY DESTINATION lib - RUNTIME DESTINATION bin -) - -# Export our configuration -install(EXPORT ccpp_framework-targets - FILE ccpp_framework-config.cmake - DESTINATION lib/cmake -) - -install(DIRECTORY ${CMAKE_Fortran_MODULE_DIRECTORY}/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) diff --git a/src/ccpp_types.F90 b/src/ccpp_types.F90 deleted file mode 100644 index cdc1d655..00000000 --- a/src/ccpp_types.F90 +++ /dev/null @@ -1,92 +0,0 @@ -! -! This work (Common Community Physics Package), identified by NOAA, NCAR, -! CU/CIRES, is free of known copyright restrictions and is placed in the -! public domain. -! -! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -! THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -! - -!> -!! @brief Type definitions module. -!! -!! @details The types module provides definitions for -!! atmospheric driver to call the CCPP. -! -module ccpp_types - - use mpi_f08, only: mpi_comm - - !! \section arg_table_ccpp_types - !! \htmlinclude ccpp_types.html - !! - - implicit none - - private - public :: ccpp_t, one - public :: mpi_comm - - !> @var Definition of constant one - integer, parameter :: one = 1 - - !> @var The default loop counter indicating outside of a subcycle loop - integer, parameter :: ccpp_default_loop_cnt = -999 - integer, parameter :: ccpp_default_loop_max = -999 - - !> @var The default values for block, chunk and thread numbers indicating invalid data - integer, parameter :: ccpp_default_block_number = -999 - integer, parameter :: ccpp_default_chunk_number = -999 - integer, parameter :: ccpp_default_thread_number = -999 - - !> @var The default maximum number of threads for CCPP - integer, parameter :: ccpp_default_thread_count = -999 - - !! \section arg_table_ccpp_t - !! \htmlinclude ccpp_t.html - !! - !> - !! @brief CCPP physics type. - !! - !! Generic type that contains all components to run the CCPP. - !! - !! - Array of fields to all the data needing to go - !! the physics drivers. - !! - The suite definitions in a ccpp_suite_t type. - ! - type :: ccpp_t - ! CCPP-internal variables for physics schemes - integer :: errflg = 0 - character(len=512) :: errmsg = '' - integer :: loop_cnt = ccpp_default_loop_cnt - integer :: loop_max = ccpp_default_loop_max - integer :: blk_no = ccpp_default_block_number - integer :: chunk_no = ccpp_default_chunk_number - integer :: thrd_no = ccpp_default_thread_number - integer :: thrd_cnt = ccpp_default_thread_count - integer :: ccpp_instance = 1 - - contains - - procedure :: initialized => ccpp_t_initialized - - end type ccpp_t - -contains - - function ccpp_t_initialized(ccpp_d) result(initialized) - implicit none - ! - class(ccpp_t) :: ccpp_d - logical :: initialized - ! - initialized = ccpp_d%thrd_no /= ccpp_default_thread_number .or. & - ccpp_d%blk_no /= ccpp_default_block_number .or. & - ccpp_d%chunk_no /= ccpp_default_chunk_number - end function ccpp_t_initialized - -end module ccpp_types diff --git a/src/ccpp_types.meta b/src/ccpp_types.meta deleted file mode 100644 index cdec1dc2..00000000 --- a/src/ccpp_types.meta +++ /dev/null @@ -1,103 +0,0 @@ -######################################################################## -[ccpp-table-properties] - name = ccpp_t - type = ddt - dependencies = - -[ccpp-arg-table] - name = ccpp_t - type = ddt -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 -[loop_cnt] - standard_name = ccpp_loop_counter - long_name = loop counter for subcycling loops in CCPP - units = index - dimensions = () - type = integer -[loop_max] - standard_name = ccpp_loop_extent - long_name = loop extent for subcycling loops in CCPP - units = count - dimensions = () - type = integer -[blk_no] - standard_name = ccpp_block_number - long_name = number of block for explicit data blocking in CCPP - units = index - dimensions = () - type = integer -[chunk_no] - standard_name = ccpp_chunk_number - long_name = number of chunk for using array chunks in CCPP - units = index - dimensions = () - type = integer -[thrd_no] - standard_name = ccpp_thread_number - long_name = number of thread for threading in CCPP - units = index - dimensions = () - type = integer -[thrd_cnt] - standard_name = ccpp_thread_count - long_name = total number of threads for threading in CCPP - units = index - dimensions = () - type = integer -[ccpp_instance] - standard_name = ccpp_instance - long_name = ccpp_instance - units = index - dimensions = () - type = integer - -######################################################################## -[ccpp-table-properties] - name = MPI_Comm - type = ddt - dependencies = - -[ccpp-arg-table] - name = MPI_Comm - type = ddt - -######################################################################## - -[ccpp-table-properties] - name = ccpp_types - type = module - dependencies = - -[ccpp-arg-table] - name = ccpp_types - type = module -[ccpp_t] - standard_name = ccpp_t - long_name = definition of type ccpp_t - units = DDT - dimensions = () - type = ccpp_t -[one] - standard_name = ccpp_constant_one - long_name = definition of constant one - units = 1 - dimensions = () - type = integer -[MPI_Comm] - standard_name = MPI_Comm - long_name = definition of type MPI_Comm - units = DDT - dimensions = () - type = MPI_Comm diff --git a/test/.pylintrc b/test/.pylintrc deleted file mode 100644 index b380843f..00000000 --- a/test/.pylintrc +++ /dev/null @@ -1,466 +0,0 @@ -[MASTER] - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. -jobs=1 - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages -suggestion-mode=yes - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=C0330,too-many-lines,too-many-public-methods,too-many-locals,too-many-arguments,too-many-instance-attributes,unnecessary-pass - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=c-extension-no-member - - -[REPORTS] - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio).You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=15 - -# Complete name of functions that never returns. When checking for -# inconsistent-return-statements if a never returning function is called then -# it will be considered as an explicit return statement and no message will be -# printed. -never-returning-functions=optparse.Values,sys.exit - - -[BASIC] - -# Naming style matching correct argument names -argument-naming-style=snake_case - -# Regular expression matching correct argument names. Overrides argument- -# naming-style -#argument-rgx= - -# Naming style matching correct attribute names -attr-naming-style=snake_case - -# Regular expression matching correct attribute names. Overrides attr-naming- -# style -#attr-rgx= - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo, - bar, - baz, - qux, - toto, - tutu, - tata - -# Naming style matching correct class attribute names -class-attribute-naming-style=any - -# Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style -#class-attribute-rgx= - -# Naming style matching correct class names -class-naming-style=PascalCase - -# Regular expression matching correct class names. Overrides class-naming-style -#class-rgx= - -# Naming style matching correct constant names -const-naming-style=UPPER_CASE - -# Regular expression matching correct constant names. Overrides const-naming- -# style -#const-rgx= - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming style matching correct function names -function-naming-style=snake_case - -# Regular expression matching correct function names. Overrides function- -# naming-style -#function-rgx= - -# Good variable names which should always be accepted, separated by a comma -good-names=_ - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# Naming style matching correct inline iteration names -inlinevar-naming-style=any - -# Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style -#inlinevar-rgx= - -# Naming style matching correct method names -method-naming-style=snake_case - -# Regular expression matching correct method names. Overrides method-naming- -# style -#method-rgx= - -# Naming style matching correct module names -module-naming-style=snake_case - -# Regular expression matching correct module names. Overrides module-naming- -# style -#module-rgx= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty - -# Naming style matching correct variable names -variable-naming-style=snake_case - -# Regular expression matching correct variable names. Overrides variable- -# naming-style -#variable-rgx= - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=(^\s*(# )??$)|(^\s*>>> .*$)||(^\s*CCPPError:) - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=0 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=100 - -# Maximum number of lines in a module -max-module-lines=2000 - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma, - dict-separator - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME, - XXX, - TODO - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes -max-spelling-suggestions=4 - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_, - _cb - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins,io,builtins - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=10 - -# Maximum number of attributes for a class (see R0902). -max-attributes=10 - -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 - -# Maximum number of branch for function / method body -max-branches=35 - -# Maximum number of locals for function / method body -max-locals=25 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of statements in function / method body -max-statements=150 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[IMPORTS] - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub, - TERMIOS, - Bastion, - rexec - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt deleted file mode 100644 index 342ddad8..00000000 --- a/test/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -add_subdirectory(utils) - -if(CCPP_FRAMEWORK_ENABLE_TESTS OR CCPP_RUN_ADVECTION_TEST) - add_subdirectory(advection_test) -endif() -if(CCPP_FRAMEWORK_ENABLE_TESTS OR CCPP_RUN_CAPGEN_TEST) - add_subdirectory(capgen_test) -endif() -if(CCPP_FRAMEWORK_ENABLE_TESTS OR CCPP_RUN_DDT_HOST_TEST) - add_subdirectory(ddthost_test) -endif() -if(CCPP_FRAMEWORK_ENABLE_TESTS OR CCPP_RUN_VAR_COMPATIBILITY_TEST) - add_subdirectory(var_compatibility_test) -endif() -if(CCPP_FRAMEWORK_ENABLE_TESTS OR CCPP_RUN_NESTED_SUITE_TEST) - add_subdirectory(nested_suite_test) -endif() diff --git a/test/README.md b/test/README.md deleted file mode 100644 index 03363532..00000000 --- a/test/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# Testing - -## Unit tests -To run the Python based unit tests, see the associated documentation in the `unit_tests` directory. - -## Doc tests -The Python source code has a wide range of doctests that can be used to verify implementation details quickly. To run the Python based doc tests, run: -```bash -$ export PYTHONPATH=/scripts:/scripts/parse_tools -$ pytest -v /scripts/ --doctest-modules -``` - -## Regression tests -The run the regression tests with mock host models, build the main project with your test option: - -```bash -$ cmake --fresh -S -B ... -$ cd -$ make -$ ctest -``` - -For example, to run all of the regression tests from the root of the project, you can use: -```bash -cmake --fresh -B./build -S./ -DCCPP_FRAMEWORK_ENABLE_TESTS=ON -``` - -Currently (as of July 2025), if everything works as expected, you should see something like: -``` -Test project - Start 1: ctest_advection_host_integration -1/4 Test #1: ctest_advection_host_integration ........... Passed 0.01 sec - Start 2: ctest_capgen_host_integration -2/4 Test #2: ctest_capgen_host_integration .............. Passed 0.01 sec - Start 3: ctest_ddt_host_integration -3/4 Test #3: ctest_ddt_host_integration ................. Passed 0.01 sec - Start 4: ctest_var_compatibility_host_integration -4/4 Test #4: ctest_var_compatibility_host_integration ... Passed 0.02 sec - -100% tests passed, 0 tests failed out of 4 - -Total Test time (real) = 0.06 sec -``` - -There are several `...` to enable tests: - -1) `-DCCPP_FRAMEWORK_ENABLE_TESTS=ON` Turns on all regression tests. -2) `-DCCPP_RUN_ADVECTION_TEST=ON` Turns on only the advection test -3) `-DCCPP_RUN_CAPGEN_TEST=ON` Turns on only the capgen test -4) `-DCCPP_RUN_DDT_HOST_TEST=ON` Turns on only the ddt host test -5) `-DCCPP_RUN_VAR_COMPATIBILITY_TEST=ON` Turns on only the variable compatibility test - -By default, the tests will build in debug mode. To enable release mode, you will need to set the build type: `-DCMAKE_BUILD_TYPE=Release` (or if you want release with debug symbols: `-DCMAKE_BUILD_TYPE=RelWithDebInfo`). - -To enable more verbose output for `ccpp_capgen.py`, add `-DCCPP_VERBOSITY=` to the `cmake` command line arguments where `n={1,2,3}` (`n=0` or no verbosity by default). - -If needed, the generated caps will be in `/test//ccpp`. - -### Python regression test interface - -There is a matching Python based API for each regression test. To run the corresponding python tests, build the framework using the build process from above and then you can run: - -```bash -BUILD_DIR= \ -PYTHONPATH=/test/:/scripts/ \ - pytest \ - /test/capgen_test/capgen_test_reports.py \ - /test/advection_test/advection_test_reports.py \ - /test/ddthost_test/ddthost_test_reports.py \ - /test/var_compatibility_test/var_compatibility_test_reports.py -``` - -You may run tests individually instead of all tests as your use case needs. - -Please see each test directory for more information on that specific test. diff --git a/test/advection_test/CMakeLists.txt b/test/advection_test/CMakeLists.txt deleted file mode 100644 index 4c20835b..00000000 --- a/test/advection_test/CMakeLists.txt +++ /dev/null @@ -1,55 +0,0 @@ -# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES -set(SCHEME_FILES "cld_liq" "cld_ice" "apply_constituent_tendencies" "const_indices") -set(SCHEME_FILES_ERROR "cld_liq" "cld_ice" "dlc_liq") -set(HOST_FILES "test_host_data" "test_host_mod") -set(SUITE_FILES "cld_suite.xml") -set(SUITE_FILES_ERROR "cld_suite_error.xml") -# HOST is the name of the executable we will build. -set(HOST "test_host") - -# By default, generated caps go in this test specific ccpp subdir -set(CCPP_CAP_FILES "${CMAKE_CURRENT_BINARY_DIR}/ccpp") - -# Create lists for Fortran and meta data files from file names -list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) -list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_META_FILES) -list(TRANSFORM SCHEME_FILES_ERROR APPEND ".F90" OUTPUT_VARIABLE SCHEME_ERROR_FORTRAN_FILES) -list(TRANSFORM SCHEME_FILES_ERROR APPEND ".meta" OUTPUT_VARIABLE SCHEME_ERROR_META_FILES) -list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE ADVECTION_HOST_FORTRAN_FILES) -list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE ADVECTION_HOST_METADATA_FILES) - -list(APPEND ADVECTION_HOST_METADATA_FILES "${HOST}.meta") - -# Run ccpp_capgen that we expect to fail -ccpp_capgen(CAPGEN_EXPECT_THROW_ERROR ON - VERBOSITY ${CCPP_VERBOSITY} - HOSTFILES ${ADVECTION_HOST_METADATA_FILES} - SCHEMEFILES ${SCHEME_ERROR_META_FILES} - SUITES ${SUITE_FILES_ERROR} - HOST_NAME ${HOST} - OUTPUT_ROOT "${CCPP_CAP_FILES}") - -# Run ccpp_capgen -ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} - HOSTFILES ${ADVECTION_HOST_METADATA_FILES} - SCHEMEFILES ${SCHEME_META_FILES} - SUITES ${SUITE_FILES} - HOST_NAME ${HOST} - OUTPUT_ROOT "${CCPP_CAP_FILES}") - -# Retrieve the list of Fortran files required for test host from datatable.xml and set to CCPP_CAPS_LIST -ccpp_datafile(DATATABLE "${CCPP_CAP_FILES}/datatable.xml" - REPORT_NAME "--ccpp-files") - -# Create test host library -add_library(ADVECTION_TESTLIB OBJECT ${SCHEME_FORTRAN_FILES} - ${ADVECTION_HOST_FORTRAN_FILES} - ${CCPP_CAPS_LIST}) - -# Setup test executable with needed dependencies -add_executable(advection_host_integration test_advection_host_integration.F90 ${HOST}.F90) -target_link_libraries(advection_host_integration PRIVATE ADVECTION_TESTLIB test_utils) -target_include_directories(advection_host_integration PRIVATE "$") - -# Add executable to be called with ctest -add_test(NAME ctest_advection_host_integration COMMAND advection_host_integration) diff --git a/test/advection_test/test_host.meta b/test/advection_test/test_host.meta deleted file mode 100644 index 5d861764..00000000 --- a/test/advection_test/test_host.meta +++ /dev/null @@ -1,38 +0,0 @@ -[ccpp-table-properties] - name = suite_info - type = ddt -[ccpp-arg-table] - name = suite_info - type = ddt - -[ccpp-table-properties] - name = test_host - type = host -[ccpp-arg-table] - name = test_host - type = host -[ col_start ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = () - protected = True -[ col_end ] - standard_name = horizontal_loop_end - type = integer - units = count - dimensions = () - protected = True -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer diff --git a/test/capgen_test/CMakeLists.txt b/test/capgen_test/CMakeLists.txt deleted file mode 100644 index 49c75842..00000000 --- a/test/capgen_test/CMakeLists.txt +++ /dev/null @@ -1,95 +0,0 @@ - -#------------------------------------------------------------------------------ -# -# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES -# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) -# -#------------------------------------------------------------------------------ -set(SCHEME_FILES "setup_coeffs" "temp_set" "temp_adjust" "temp_calc_adjust") -set(SUITE_SCHEME_FILES "make_ddt" "environ_conditions" "temp_kinds") -set(HOST_FILES "test_host_data" "test_host_mod") -set(SUITE_FILES "ddt_suite.xml" "temp_suite.xml") -set(KIND_TYPE "kind_phys=REAL64") - -# HOST is the name of the executable we will build. -# We assume there are files ${HOST}.meta and ${HOST}.F90 in CMAKE_SOURCE_DIR -set(HOST "test_host") - -# By default, generated caps go in ccpp subdir -set(CCPP_CAP_FILES "${CMAKE_CURRENT_BINARY_DIR}/ccpp") - -# Create lists for Fortran and meta data files from file names -# Fortran files are not all in one directory -set(SCHEME_FORTRAN_FILES "") -foreach(sfile ${SCHEME_FILES}) - find_file(fort_file "${sfile}.F90" NO_CACHE - HINTS ${CMAKE_CURRENT_SOURCE_DIR} - HINTS ${CMAKE_CURRENT_SOURCE_DIR}/source_dir1 - HINTS ${CMAKE_CURRENT_SOURCE_DIR}/source_dir2) - list(APPEND SCHEME_FORTRAN_FILES ${fort_file}) - unset(fort_file) -endforeach() -find_file(fort_file "ddt2.F90" NO_CACHE - HINTS ${CMAKE_CURRENT_SOURCE_DIR}) -list(APPEND SCHEME_FORTRAN_FILES ${fort_file}) -unset(fort_file) -list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_META_FILES) -set(SUITE_SCHEME_FORTRAN_FILES "") -foreach(sfile ${SUITE_SCHEME_FILES}) - find_file(fort_file "${sfile}.F90" NO_CACHE - HINTS ${CMAKE_CURRENT_SOURCE_DIR} - HINTS ${CMAKE_CURRENT_SOURCE_DIR}/source_dir1 - HINTS ${CMAKE_CURRENT_SOURCE_DIR}/source_dir2 - HINTS ${CMAKE_CURRENT_SOURCE_DIR}/adjust) - list(APPEND SUITE_SCHEME_FORTRAN_FILES ${fort_file}) - unset(fort_file) -endforeach() - -list(TRANSFORM SUITE_SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SUITE_SCHEME_META_FILES) -list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE CAPGEN_HOST_FORTRAN_FILES) -list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE CAPGEN_HOST_METADATA_FILES) - -list(APPEND CAPGEN_HOST_METADATA_FILES "${HOST}.meta") - -ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} - HOSTFILES ${CAPGEN_HOST_METADATA_FILES} - SCHEMEFILES ${SCHEME_META_FILES} ${SUITE_SCHEME_META_FILES} - SUITES ${SUITE_FILES} - HOST_NAME ${HOST} - KIND_TYPES ${KIND_TYPES} - OUTPUT_ROOT "${CCPP_CAP_FILES}") - -# Retrieve the list of Fortran files required for test host from datatable.xml and set to CCPP_CAPS_LIST -ccpp_datafile(DATATABLE "${CCPP_CAP_FILES}/datatable.xml" - REPORT_NAME "--ccpp-files") - -# Create test host library -add_library(CAPGEN_TESTLIB OBJECT ${SCHEME_FORTRAN_FILES} - ${SUITE_SCHEME_FORTRAN_FILES} - ${CAPGEN_HOST_FORTRAN_FILES} - ${CCPP_CAPS_LIST}) - -# Setup test executable with needed dependencies -add_executable(capgen_host_integration test_capgen_host_integration.F90 ${HOST}.F90) -if(OPENMP) - target_link_libraries(capgen_host_integration PRIVATE OpenMP::OpenMP_Fortran) -endif() -target_link_libraries(capgen_host_integration PRIVATE CAPGEN_TESTLIB test_utils) -target_include_directories(capgen_host_integration PRIVATE "$") - -# Add executable to be called with ctest -add_test(NAME ctest_capgen_host_integration_omp1 - COMMAND capgen_host_integration) - -add_test(NAME ctest_capgen_host_integration_omp2 - COMMAND capgen_host_integration) - -set_tests_properties(ctest_capgen_host_integration_omp1 - PROPERTIES - ENVIRONMENT "OMP_NUM_THREADS=1" -) - -set_tests_properties(ctest_capgen_host_integration_omp2 - PROPERTIES - ENVIRONMENT "OMP_NUM_THREADS=2" -) diff --git a/test/capgen_test/test_capgen_host_integration.F90 b/test/capgen_test/test_capgen_host_integration.F90 deleted file mode 100644 index 7f964178..00000000 --- a/test/capgen_test/test_capgen_host_integration.F90 +++ /dev/null @@ -1,89 +0,0 @@ -program test - use test_prog, only: test_host, & - suite_info, & - cm, & - cs - - implicit none - - character(len=cs), target :: test_parts1(2) = (/ 'physics1 ', & - 'physics2 ' /) - character(len=cs), target :: test_parts2(1) = (/ 'data_prep ' /) - character(len=cm), target :: test_invars1(10) = (/ & - 'potential_temperature ', & - 'potential_temperature_at_interface ', & - 'coefficients_for_interpolation ', & - 'surface_air_pressure ', & - 'water_vapor_specific_humidity ', & - 'potential_temperature_increment ', & - 'soil_levels ', & - 'temperature_at_diagnostic_levels ', & - 'time_step_for_physics ', & - 'array_variable_for_testing ' /) - character(len=cm), target :: test_outvars1(10) = (/ & - 'potential_temperature ', & - 'potential_temperature_at_interface ', & - 'coefficients_for_interpolation ', & - 'surface_air_pressure ', & - 'water_vapor_specific_humidity ', & - 'soil_levels ', & - 'temperature_at_diagnostic_levels ', & - 'ccpp_error_code ', & - 'ccpp_error_message ', & - 'array_variable_for_testing ' /) - character(len=cm), target :: test_reqvars1(12) = (/ & - 'potential_temperature ', & - 'potential_temperature_at_interface ', & - 'coefficients_for_interpolation ', & - 'surface_air_pressure ', & - 'water_vapor_specific_humidity ', & - 'potential_temperature_increment ', & - 'time_step_for_physics ', & - 'soil_levels ', & - 'temperature_at_diagnostic_levels ', & - 'ccpp_error_code ', & - 'ccpp_error_message ', & - 'array_variable_for_testing ' /) - - character(len=cm), target :: test_invars2(3) = (/ & - 'model_times ', & - 'number_of_model_times ', & - 'surface_air_pressure ' /) - - character(len=cm), target :: test_outvars2(5) = (/ & - 'ccpp_error_code ', & - 'ccpp_error_message ', & - 'model_times ', & - 'surface_air_pressure ', & - 'number_of_model_times ' /) - - character(len=cm), target :: test_reqvars2(5) = (/ & - 'model_times ', & - 'number_of_model_times ', & - 'surface_air_pressure ', & - 'ccpp_error_code ', & - 'ccpp_error_message ' /) - type(suite_info) :: test_suites(2) - logical :: run_okay - - ! Setup expected test suite info - test_suites(1)%suite_name = 'temp_suite' - test_suites(1)%suite_parts => test_parts1 - test_suites(1)%suite_input_vars => test_invars1 - test_suites(1)%suite_output_vars => test_outvars1 - test_suites(1)%suite_required_vars => test_reqvars1 - test_suites(2)%suite_name = 'ddt_suite' - test_suites(2)%suite_parts => test_parts2 - test_suites(2)%suite_input_vars => test_invars2 - test_suites(2)%suite_output_vars => test_outvars2 - test_suites(2)%suite_required_vars => test_reqvars2 - - call test_host(run_okay, test_suites) - - if (run_okay) then - stop 0 - else - stop -1 - end if - -end program test diff --git a/test/capgen_test/test_host.meta b/test/capgen_test/test_host.meta deleted file mode 100644 index 5d861764..00000000 --- a/test/capgen_test/test_host.meta +++ /dev/null @@ -1,38 +0,0 @@ -[ccpp-table-properties] - name = suite_info - type = ddt -[ccpp-arg-table] - name = suite_info - type = ddt - -[ccpp-table-properties] - name = test_host - type = host -[ccpp-arg-table] - name = test_host - type = host -[ col_start ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = () - protected = True -[ col_end ] - standard_name = horizontal_loop_end - type = integer - units = count - dimensions = () - protected = True -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer diff --git a/test/ddthost_test/CMakeLists.txt b/test/ddthost_test/CMakeLists.txt deleted file mode 100644 index 5516277b..00000000 --- a/test/ddthost_test/CMakeLists.txt +++ /dev/null @@ -1,53 +0,0 @@ - -#------------------------------------------------------------------------------ -# -# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES -# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) -# -#------------------------------------------------------------------------------ -set(SCHEME_FILES "setup_coeffs" "temp_set" "temp_adjust" "temp_calc_adjust") -set(SUITE_SCHEME_FILES "make_ddt" "environ_conditions") -set(HOST_FILES "test_host_data" "test_host_mod" "host_ccpp_ddt") -set(SUITE_FILES "ddt_suite.xml" "temp_suite.xml") -# HOST is the name of the executable we will build. -# We assume there are files ${HOST}.meta and ${HOST}.F90 in CMAKE_SOURCE_DIR -set(HOST "test_host") - -# By default, generated caps go in ccpp subdir -set(CCPP_CAP_FILES "${CMAKE_CURRENT_BINARY_DIR}/ccpp") - -# Create lists for Fortran and meta data files from file names -list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) -list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_META_FILES) -list(TRANSFORM SUITE_SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SUITE_SCHEME_FORTRAN_FILES) -list(TRANSFORM SUITE_SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SUITE_SCHEME_META_FILES) -list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE DDT_HOST_FORTRAN_FILES) -list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE DDT_HOST_METADATA_FILES) - -list(APPEND DDT_HOST_METADATA_FILES "${HOST}.meta") - -# Run ccpp_capgen -ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} - HOSTFILES ${DDT_HOST_METADATA_FILES} - SCHEMEFILES ${SCHEME_META_FILES} ${SUITE_SCHEME_META_FILES} - SUITES ${SUITE_FILES} - HOST_NAME ${HOST} - OUTPUT_ROOT "${CCPP_CAP_FILES}") - -# Retrieve the list of Fortran files required for test host from datatable.xml and set to CCPP_CAPS_LIST -ccpp_datafile(DATATABLE "${CCPP_CAP_FILES}/datatable.xml" - REPORT_NAME "--ccpp-files") - -# Create test host library -add_library(DDT_TESTLIB OBJECT ${SCHEME_FORTRAN_FILES} - ${SUITE_SCHEME_FORTRAN_FILES} - ${DDT_HOST_FORTRAN_FILES} - ${CCPP_CAPS_LIST}) - -# Setup test executable with needed dependencies -add_executable(ddt_host_integration test_ddt_host_integration.F90 ${HOST}.F90) -target_link_libraries(ddt_host_integration PRIVATE DDT_TESTLIB test_utils) -target_include_directories(ddt_host_integration PRIVATE "$") - -# Add executable to be called with ctest -add_test(NAME ctest_ddt_host_integration COMMAND ddt_host_integration) diff --git a/test/ddthost_test/test_ddt_host_integration.F90 b/test/ddthost_test/test_ddt_host_integration.F90 deleted file mode 100644 index c3cef458..00000000 --- a/test/ddthost_test/test_ddt_host_integration.F90 +++ /dev/null @@ -1,82 +0,0 @@ -program test - use test_prog, only: test_host, & - suite_info, & - cm, & - cs - - implicit none - - character(len=cs), target :: test_parts1(2) = (/ 'physics1 ', & - 'physics2 ' /) - character(len=cs), target :: test_parts2(1) = (/ 'data_prep ' /) - character(len=cm), target :: test_invars1(7) = (/ & - 'potential_temperature ', & - 'potential_temperature_at_interface ', & - 'coefficients_for_interpolation ', & - 'surface_air_pressure ', & - 'water_vapor_specific_humidity ', & - 'potential_temperature_increment ', & - 'time_step_for_physics ' /) - character(len=cm), target :: test_outvars1(7) = (/ & - 'potential_temperature ', & - 'potential_temperature_at_interface ', & - 'coefficients_for_interpolation ', & - 'surface_air_pressure ', & - 'water_vapor_specific_humidity ', & - 'ccpp_error_code ', & - 'ccpp_error_message ' /) - character(len=cm), target :: test_reqvars1(9) = (/ & - 'potential_temperature ', & - 'potential_temperature_at_interface ', & - 'coefficients_for_interpolation ', & - 'surface_air_pressure ', & - 'water_vapor_specific_humidity ', & - 'potential_temperature_increment ', & - 'time_step_for_physics ', & - 'ccpp_error_code ', & - 'ccpp_error_message ' /) - - character(len=cm), target :: test_invars2(4) = (/ & - 'model_times ', & - 'number_of_model_times ', & - 'surface_air_pressure ', & - 'host_standard_ccpp_type ' /) - - character(len=cm), target :: test_outvars2(5) = (/ & - 'ccpp_error_code ', & - 'ccpp_error_message ', & - 'model_times ', & - 'surface_air_pressure ', & - 'number_of_model_times ' /) - - character(len=cm), target :: test_reqvars2(6) = (/ & - 'model_times ', & - 'number_of_model_times ', & - 'surface_air_pressure ', & - 'ccpp_error_code ', & - 'ccpp_error_message ', & - 'host_standard_ccpp_type ' /) - type(suite_info) :: test_suites(2) - logical :: run_okay - - ! Setup expected test suite info - test_suites(1)%suite_name = 'temp_suite' - test_suites(1)%suite_parts => test_parts1 - test_suites(1)%suite_input_vars => test_invars1 - test_suites(1)%suite_output_vars => test_outvars1 - test_suites(1)%suite_required_vars => test_reqvars1 - test_suites(2)%suite_name = 'ddt_suite' - test_suites(2)%suite_parts => test_parts2 - test_suites(2)%suite_input_vars => test_invars2 - test_suites(2)%suite_output_vars => test_outvars2 - test_suites(2)%suite_required_vars => test_reqvars2 - - call test_host(run_okay, test_suites) - - if (run_okay) then - stop 0 - else - stop -1 - end if - -end program test diff --git a/test/ddthost_test/test_host.meta b/test/ddthost_test/test_host.meta deleted file mode 100644 index 82fdc462..00000000 --- a/test/ddthost_test/test_host.meta +++ /dev/null @@ -1,18 +0,0 @@ -[ccpp-table-properties] - name = suite_info - type = ddt -[ccpp-arg-table] - name = suite_info - type = ddt - -[ccpp-table-properties] - name = test_host - type = host -[ccpp-arg-table] - name = test_host - type = host -[ ccpp ] - standard_name = host_standard_ccpp_type - type = ccpp_info_t - dimensions = () - protected = False diff --git a/test/hash_table_tests/Makefile b/test/hash_table_tests/Makefile deleted file mode 100644 index 5db42275..00000000 --- a/test/hash_table_tests/Makefile +++ /dev/null @@ -1,38 +0,0 @@ -SHELL = /bin/sh - -INCFLAG = -I -INCPATH += $(INCFLAG). -FCFLAGS += -g - -SRCPATH = ../../src -HASHPATH = $(SRCPATH) - -HASHOBJS = ccpp_hashable.o ccpp_hash_table.o - -# Make sure we have a log file -ifeq ($(LOGFILE),) -LOGFILE := ccpp_test.log -endif - -# TARGETS - -ccpp_hashable.o: $(HASHPATH)/ccpp_hashable.F90 - @echo "${FC} -c ${FCFLAGS} ${INCPATH} $^" 2>&1 >> $(LOGFILE) - @${FC} -c ${FCFLAGS} ${INCPATH} $^ 2>&1 >> $(LOGFILE) - -ccpp_hash_table.o: $(HASHPATH)/ccpp_hash_table.F90 - @echo "${FC} -c ${FCFLAGS} ${INCPATH} $^" 2>&1 >> $(LOGFILE) - @${FC} -c ${FCFLAGS} ${INCPATH} $^ 2>&1 >> $(LOGFILE) - -test_hash_table: test_hash.F90 $(HASHOBJS) - @echo "${FC} ${FCFLAGS} ${INCPATH} -o $@ $^" 2>&1 >> $(LOGFILE) - @${FC} ${FCFLAGS} ${INCPATH} -o $@ $^ 2>&1 >> $(LOGFILE) - -test: test_hash_table - @echo "Run Hash Table Tests" - @./test_hash_table - -# CLEAN -clean: - @rm -f *.o *.mod ccpp_test.log - @rm -f test_hash_table diff --git a/test/hash_table_tests/test_hash.F90 b/test/hash_table_tests/test_hash.F90 deleted file mode 100644 index b7faa074..00000000 --- a/test/hash_table_tests/test_hash.F90 +++ /dev/null @@ -1,218 +0,0 @@ -module test_hash_utils - use ccpp_hashable, only: ccpp_hashable_char_t - - implicit none - private - - public :: test_table - - integer, parameter, public :: max_terrs = 16 - - type, public :: hash_object_t - type(ccpp_hashable_char_t), pointer :: item => null() - end type hash_object_t - - private add_error - -contains - - subroutine add_error(msg, num_errs, errors) - ! Dummy arguments - character(len=*), intent(in) :: msg - integer, intent(inout) :: num_errs - character(len=*), intent(inout) :: errors(:) - - if (num_errs < max_terrs) then - num_errs = num_errs + 1 - write(errors(num_errs), *) trim(msg) - end if - - end subroutine add_error - - subroutine test_table(hash_table, table_size, num_tests, num_errs, errors) - use ccpp_hash_table, only: ccpp_hash_table_t, & - ccpp_hash_iterator_t - use ccpp_hashable, only: ccpp_hashable_t, & - new_hashable_char - - ! Dummy arguments - type(ccpp_hash_table_t), target, intent(inout) :: hash_table - integer, intent(in) :: table_size - integer, intent(out) :: num_tests - integer, intent(out) :: num_errs - character(len=*), intent(inout) :: errors(:) - ! Local variables - integer, parameter :: num_test_entries = 4 - integer, parameter :: key_len = 10 - character(len=key_len) :: hash_names(num_test_entries) = (/ & - 'foo ', 'bar ', 'foobar ', 'big daddy ' /) - logical :: hash_found(num_test_entries) - - type(hash_object_t) :: hash_chars(num_test_entries) - class(ccpp_hashable_t), pointer :: test_ptr => null() - type(ccpp_hash_iterator_t) :: hash_iter - character(len=key_len) :: test_key - character(len=len(errors(1))) :: errmsg - integer :: index - - write(6, '(a,i0)') "Testing hash table, size = ", table_size - num_tests = 0 - num_errs = 0 - ! Make sure hash table is *not* initialized - if (hash_table%is_initialized()) then - call add_error("Error: hash table initialized too early", & - num_errs, errors) - end if - num_tests = num_tests + 1 - ! Initialize hash table - call hash_table%initialize(table_size) - ! Make sure hash table is *is* initialized - if (.not. hash_table%is_initialized()) then - call add_error("Error: hash table *not* initialized", num_errs, errors) - end if - num_tests = num_tests + 1 - do index = 1, num_test_entries - call new_hashable_char(hash_names(index), hash_chars(index)%item) - call hash_table%add_hash_key(hash_chars(index)%item, & - errmsg=errors(num_errs + 1)) - if (len_trim(errors(num_errs + 1)) > 0) then - num_errs = num_errs + 1 - end if - if (num_errs > max_terrs) then - exit - end if - end do - - if (num_errs == 0) then - ! We have populated the table, let's do some tests - ! First, make sure we can find existing entries - do index = 1, num_test_entries - test_ptr => hash_table%table_value(hash_names(index), & - errmsg=errors(num_errs + 1)) - if (len_trim(errors(num_errs + 1)) > 0) then - num_errs = num_errs + 1 - else if (trim(test_ptr%key()) /= trim(hash_names(index))) then - num_errs = num_errs + 1 - write(errmsg, *) "ERROR: Found '", trim(test_ptr%key()), & - "', expected '", trim(hash_names(index)), "'" - call add_error(trim(errmsg), num_errs, errors) - end if - if (num_errs > max_terrs) then - exit - end if - end do - num_tests = num_tests + 1 - ! Next, make sure we do not find a non-existent entry - test_ptr => hash_table%table_value(trim(hash_names(1)) // '_oops', & - errmsg=errors(num_errs + 1)) - if (len_trim(errors(num_errs + 1)) == 0) then - write(errmsg, *) "ERROR: Found an entry for '", & - trim(hash_names(1)) // '_oops', "'" - call add_error(trim(errmsg), num_errs, errors) - end if - num_tests = num_tests + 1 - ! Make sure we get an error if we try to add a duplicate key - call hash_table%add_hash_key(hash_chars(2)%item, & - errmsg=errors(num_errs + 1)) - if (len_trim(errors(num_errs + 1)) == 0) then - num_errs = num_errs + 1 - write(errors(num_errs), *) & - "ERROR: Allowed duplicate entry for '", & - hash_chars(2)%item%key(), "'" - end if - num_tests = num_tests + 1 - ! Check that the total number of table entries is correct - if (hash_table%num_values() /= num_test_entries) then - write(errmsg, '(2(a,i0))') "ERROR: Wrong table value count, ", & - hash_table%num_values(), ', should be ', num_test_entries - call add_error(errmsg, num_errs, errors) - end if - num_tests = num_tests + 1 - ! Test iteration through hash table - hash_found(:) = .false. - call hash_iter%initialize(hash_table) - num_tests = num_tests + 1 - do - if (hash_iter%valid()) then - test_key = hash_iter%key() - index = 1 - do - if (trim(test_key) == trim(hash_names(index))) then - hash_found(index) = .true. - exit - else if (index >= num_test_entries) then - write(errmsg, '(3a)') & - "ERROR: Unexpected table entry, '", & - trim(test_key), "'" - call add_error(errmsg, num_errs, errors) - end if - index = index + 1 - end do - call hash_iter%next() - else - exit - end if - end do - call hash_iter%finalize() - if (any(.not. hash_found)) then - write(errmsg, '(a,i0,a)') "ERROR: ", & - count(.not. hash_found), " test keys not found in table." - call add_error(errmsg, num_errs, errors) - end if - end if - ! Finally, clear the hash table (should deallocate everything) - call hash_table%clear() - ! Make sure hash table is *not* initialized - if (hash_table%is_initialized()) then - call add_error("Error: hash table initialized after clear", & - num_errs, errors) - end if - num_tests = num_tests + 1 - ! Cleanup - do index = 1, num_test_entries - deallocate(hash_chars(index)%item) - end do - - end subroutine test_table - -end module test_hash_utils - -program test_hash - use ccpp_hash_table, only: ccpp_hash_table_t - use test_hash_utils, only: test_table, & - max_terrs - - integer, parameter :: num_table_sizes = 5 - integer, parameter :: max_errs = max_terrs * num_table_sizes - integer, parameter :: err_size = 128 - integer, parameter :: test_sizes(num_table_sizes) = (/ & - 0, 1, 2, 4, 20 /) - - type(ccpp_hash_table_t), target :: hash_table - integer :: index - integer :: errcnt = 0 - integer :: num_tests = 0 - integer :: total_errcnt = 0 - integer :: total_tests = 0 - character(len=err_size) :: errors(max_errs) - - errors = '' - do index = 1, num_table_sizes - call test_table(hash_table, test_sizes(index), num_tests, errcnt, & - errors(total_errcnt + 1:)) - total_tests = total_tests + num_tests - total_errcnt = total_errcnt + errcnt - end do - - if (total_errcnt > 0) then - write(6, '(a,i0,a)') 'FAIL, ', total_errcnt, ' errors found' - do index = 1, total_errcnt - write(6, *) trim(errors(index)) - end do - stop 1 - else - write(6, '(a,i0,a)') "All ", total_tests, " hash table tests passed!" - stop 0 - end if - -end program test_hash diff --git a/test/nested_suite_test/CMakeLists.txt b/test/nested_suite_test/CMakeLists.txt deleted file mode 100644 index c55d9bed..00000000 --- a/test/nested_suite_test/CMakeLists.txt +++ /dev/null @@ -1,49 +0,0 @@ - -#------------------------------------------------------------------------------ -# -# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES -# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) -# -#------------------------------------------------------------------------------ -set(SCHEME_FILES "effr_calc" "effrs_calc" "effr_diag" "effr_pre" "effr_post" "rad_lw" "rad_sw") -set(HOST_FILES "module_rad_ddt" "test_host_data" "test_host_mod") -set(SUITE_FILES "main_suite.xml") -# HOST is the name of the executable we will build. -# We assume there are files ${HOST}.meta and ${HOST}.F90 in CMAKE_SOURCE_DIR -set(HOST "test_host") - -# By default, generated caps go in ccpp subdir -set(CCPP_CAP_FILES "${CMAKE_CURRENT_BINARY_DIR}/ccpp") - -# Create lists for Fortran and meta data files from file names -list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) -list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_META_FILES) -list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE NESTED_SUITE_HOST_FORTRAN_FILES) -list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE NESTED_SUITE_HOST_METADATA_FILES) - -list(APPEND NESTED_SUITE_HOST_METADATA_FILES "${HOST}.meta") - -# Run ccpp_capgen -ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} - HOSTFILES ${NESTED_SUITE_HOST_METADATA_FILES} - SCHEMEFILES ${SCHEME_META_FILES} - SUITES ${SUITE_FILES} - HOST_NAME ${HOST} - OUTPUT_ROOT "${CCPP_CAP_FILES}") - -# Retrieve the list of Fortran files required for test host from datatable.xml and set to CCPP_CAPS_LIST -ccpp_datafile(DATATABLE "${CCPP_CAP_FILES}/datatable.xml" - REPORT_NAME "--ccpp-files") - -# Create test host library -add_library(NESTED_SUITE_TESTLIB OBJECT ${SCHEME_FORTRAN_FILES} - ${NESTED_SUITE_HOST_FORTRAN_FILES} - ${CCPP_CAPS_LIST}) - -# Setup test executable with needed dependencies -add_executable(nested_suite_host_integration test_nested_suite_integration.F90 ${HOST}.F90) -target_link_libraries(nested_suite_host_integration PRIVATE NESTED_SUITE_TESTLIB test_utils) -target_include_directories(nested_suite_host_integration PRIVATE "$") - -# Add executable to be called with ctest -add_test(NAME ctest_nested_suite_host_integration COMMAND nested_suite_host_integration) diff --git a/test/nested_suite_test/ccpp_kinds.F90 b/test/nested_suite_test/ccpp_kinds.F90 deleted file mode 100644 index 2eed03c9..00000000 --- a/test/nested_suite_test/ccpp_kinds.F90 +++ /dev/null @@ -1,27 +0,0 @@ -! -! This work (Common Community Physics Package Framework), identified by -! NOAA, NCAR, CU/CIRES, is free of known copyright restrictions and is -! placed in the public domain. -! -! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -! THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -!> -!! @brief Auto-generated kinds for CCPP -!! -! -module ccpp_kinds - - use iso_fortran_env, only: & - kind_phys => real64 - - implicit none - private - - public :: kind_phys - -end module ccpp_kinds diff --git a/test/nested_suite_test/test_host.meta b/test/nested_suite_test/test_host.meta deleted file mode 100644 index da71b182..00000000 --- a/test/nested_suite_test/test_host.meta +++ /dev/null @@ -1,38 +0,0 @@ -[ccpp-table-properties] - name = suite_info - type = ddt -[ccpp-arg-table] - name = suite_info - type = ddt - -[ccpp-table-properties] - name = test_host - type = host -[ccpp-arg-table] - name = test_host - type = host -[ col_start ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = () - protected = True -[ col_end ] - standard_name = horizontal_loop_end - type = integer - units = count - dimensions = () - protected = True -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = None - dimensions = () - type = character - kind = len=512 -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer diff --git a/test/pylint_test.sh b/test/pylint_test.sh deleted file mode 100755 index a8bf3f90..00000000 --- a/test/pylint_test.sh +++ /dev/null @@ -1,28 +0,0 @@ -#! /bin/bash - -# Script to run pylint tests on CCPP Framework python scripts - -# Add CCPP Framework paths to PYTHONPATH so pylint can find them -SCRIPTDIR="$( cd $( dirname ${0} ); pwd -P )" -SPINROOT="$( dirname ${SCRIPTDIR} )" -CCPPDIR="${SPINROOT}/scripts" -export PYTHONPATH="${CCPPDIR}:$PYTHONPATH" - -pylintcmd="pylint --rcfile=${SCRIPTDIR}/.pylintrc" - -# Test top-level scripts -scripts="${CCPPDIR}/ccpp_capgen.py" -scripts="${scripts} ${CCPPDIR}/ccpp_suite.py" -scripts="${scripts} ${CCPPDIR}/ddt_library.py" -scripts="${scripts} ${CCPPDIR}/host_cap.py" -scripts="${scripts} ${CCPPDIR}/host_model.py" -scripts="${scripts} ${CCPPDIR}/metadata_table.py" -scripts="${scripts} ${CCPPDIR}/metavar.py" -scripts="${scripts} ${CCPPDIR}/state_machine.py" -${pylintcmd} ${scripts} -# Test the fortran_tools module -${pylintcmd} ${CCPPDIR}/fortran_tools -# Test the parse_tools module -${pylintcmd} ${CCPPDIR}/parse_tools -# Test the fortran to metadata converter tool -${pylintcmd} ${CCPPDIR}/ccpp_fortran_to_metadata.py diff --git a/test/test_fortran_to_metadata.sh b/test/test_fortran_to_metadata.sh deleted file mode 100755 index adedaac6..00000000 --- a/test/test_fortran_to_metadata.sh +++ /dev/null @@ -1,37 +0,0 @@ -#! /bin/bash - -## Relevant directories and file paths -test_dir="$(cd $(dirname ${0}); pwd -P)" -script_dir="$(dirname ${test_dir})/scripts" -sample_files_dir="${test_dir}/unit_tests/sample_files" -f2m_script="${script_dir}/ccpp_fortran_to_metadata.py" -filename="test_fortran_to_metadata" -test_input="${sample_files_dir}/${filename}.F90" -tmp_dir="${test_dir}/unit_tests/tmp" -sample_meta="${sample_files_dir}/check_fortran_to_metadata.meta" - -# Run the script -opts="--ddt-names serling_t" -${f2m_script} --output-root "${tmp_dir}" ${opts} "${test_input}" -res=$? - -retval=0 -if [ ${res} -ne 0 ]; then - echo "FAIL: ccpp_fortran_to_metadata.py exited with error ${res}" - retval=${res} -elif [ ! -f "${tmp_dir}/${filename}.meta" ]; then - echo "FAIL: metadata file, '${tmp_dir}/${filename}.meta', not created" - retval=1 -else - cmp --quiet "${sample_meta}" "${tmp_dir}/${filename}.meta" - res=$? - if [ ${res} -ne 0 ]; then - echo "FAIL: Comparison with correct metadata file failed" - retval=${res} - else - echo "PASS" - # Cleanup - rm "${tmp_dir}/${filename}.meta" - fi -fi -exit ${retval} diff --git a/test/test_offline_metadata_checker.sh b/test/test_offline_metadata_checker.sh deleted file mode 100755 index 90befbec..00000000 --- a/test/test_offline_metadata_checker.sh +++ /dev/null @@ -1,35 +0,0 @@ -#! /bin/bash - -## Relevant directories and file paths -root_dir="$(cd $(dirname ${0}); pwd -P)" -script_dir="$(dirname ${root_dir})/scripts/fortran_tools" -test_dir="$(dirname ${root_dir})/test/advection_test" -offline_script="${script_dir}/offline_check_fortran_vs_metadata.py" -relative_path="capgen_test" - -# Run the script -${offline_script} --directory ${test_dir} -res=$? - -retval=0 -if [ ${res} -ne 0 ]; then - echo "FAIL: offline_check_fortran_vs_metadata.py exited with error ${res} while checking ${test_dir}" - retval=${res} - exit ${retval} -else - echo "PASS" -fi - -# Run the script again with a relative path -cd ${root_dir} -${offline_script} --directory ${relative_path} -res=$? - -retval=0 -if [ ${res} -ne 0 ]; then - echo "FAIL: offline_check_fortran_vs_metadata.py exited with error ${res} while checking ${relative_path}" - retval=${res} -else - echo "PASS" -fi -exit ${retval} diff --git a/test/test_stub.py b/test/test_stub.py deleted file mode 100644 index 664f3277..00000000 --- a/test/test_stub.py +++ /dev/null @@ -1,162 +0,0 @@ -from ccpp_datafile import datatable_report, DatatableReport -import subprocess - - -class BaseTests: - - - class TestHostDataTables: - _SEP = "," - - def test_host_files(self): - test_str = datatable_report(self.database, DatatableReport("host_files"), self._SEP) - self.assertSetEqual(set(self.host_files), set(test_str.split(self._SEP))) - - def test_suite_files(self): - test_str = datatable_report(self.database, DatatableReport("suite_files"), self._SEP) - self.assertSetEqual(set(self.suite_files), set(test_str.split(self._SEP))) - - def test_utility_files(self): - test_str = datatable_report(self.database, DatatableReport("utility_files"), self._SEP) - self.assertSetEqual(set(self.utility_files), set(test_str.split(self._SEP))) - - def test_ccpp_files(self): - test_str = datatable_report(self.database, DatatableReport("ccpp_files"), self._SEP) - self.assertSetEqual(set(self.ccpp_files), set(test_str.split(self._SEP))) - - def test_process_list(self): - test_str = datatable_report(self.database, DatatableReport("process_list"), self._SEP) - self.assertSetEqual(set(self.process_list), set(test_str.split(self._SEP))) - - def test_module_list(self): - test_str = datatable_report(self.database, DatatableReport("module_list"), self._SEP) - self.assertSetEqual(set(self.module_list), set(test_str.split(self._SEP))) - - def test_dependencies_list(self): - test_str = datatable_report(self.database, DatatableReport("dependencies"), self._SEP) - self.assertSetEqual(set(self.dependencies), set(test_str.split(self._SEP))) - - def test_suite_list(self): - test_str = datatable_report(self.database, DatatableReport("suite_list"), self._SEP) - self.assertSetEqual(set(self.suite_list), set(test_str.split(self._SEP))) - - - class TestHostCommandLineDataFiles: - _SEP = "," - - def test_host_files(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--host-files"], - capture_output=True, - text=True) - self.assertEqual(self._SEP.join(self.host_files), completedProcess.stdout.strip()) - - def test_suite_files(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--suite-files"], - capture_output=True, - text=True) - self.assertEqual(self._SEP.join(self.suite_files), completedProcess.stdout.strip()) - - def test_utility_files(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--utility-files"], - capture_output=True, - text=True) - self.assertEqual(self._SEP.join(self.utility_files), completedProcess.stdout.strip()) - - def test_ccpp_files(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--ccpp-files"], - capture_output=True, - text=True) - self.assertEqual(self._SEP.join(self.ccpp_files), completedProcess.stdout.strip()) - - def test_process_list(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--process-list"], - capture_output=True, - text=True) - actualOutput = {s.strip() for s in completedProcess.stdout.split(self._SEP)} - self.assertSetEqual(set(self.process_list), actualOutput) - - def test_module_list(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--module-list"], - capture_output=True, - text=True) - actualOutput = {s.strip() for s in completedProcess.stdout.split(self._SEP)} - self.assertSetEqual(set(self.module_list), actualOutput) - - def test_dependencies(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--dependencies"], - capture_output=True, - text=True) - self.assertEqual(set(self.dependencies), set(completedProcess.stdout.strip().split(self._SEP))) - - def test_suite_list(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--suite-list"], - capture_output=True, - text=True) - self.assertEqual(self._SEP.join(self.suite_list), completedProcess.stdout.strip()) - - - class TestSuite: - _SEP = "," - - def test_required_variables(self): - test_str = datatable_report(self.database, DatatableReport("required_variables", value=self.suite_name), self._SEP) - self.assertSetEqual(set(self.required_vars), set(test_str.split(self._SEP))) - - def test_input_variables(self): - test_str = datatable_report(self.database, DatatableReport("input_variables", value=self.suite_name), self._SEP) - self.assertSetEqual(set(self.input_vars), set(test_str.split(self._SEP))) - - def test_output_variables(self): - test_str = datatable_report(self.database, DatatableReport("output_variables", value=self.suite_name), self._SEP) - self.assertSetEqual(set(self.output_vars), set(test_str.split(self._SEP))) - - - class TestSuiteCommandLine: - _SEP = "," - - def test_required_variables(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--required-variables", self.suite_name], - capture_output=True, - text=True) - actualOutput = {s.strip() for s in completedProcess.stdout.split(self._SEP)} - self.assertSetEqual(set(self.required_vars), actualOutput) - - def test_input_variables(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--input-variables", self.suite_name], - capture_output=True, - text=True) - actualOutput = {s.strip() for s in completedProcess.stdout.split(self._SEP)} - self.assertSetEqual(set(self.input_vars), actualOutput) - - def test_output_variables(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--output-variables", self.suite_name], - capture_output=True, - text=True) - actualOutput = {s.strip() for s in completedProcess.stdout.split(self._SEP)} - self.assertSetEqual(set(self.output_vars), actualOutput) - - - class TestSuiteExcludeProtected(TestSuite): - def test_required_variables_excluding_protected(self): - test_str = datatable_report(self.database, DatatableReport("required_variables", value="temp_suite"), self._SEP, exclude_protected=True) - self.assertSetEqual(set(self.required_vars_excluding_protected), set(test_str.split(self._SEP))) - - def test_input_variables_excluding_protected(self): - test_str = datatable_report(self.database, DatatableReport("input_variables", value="temp_suite"), self._SEP, exclude_protected=True) - self.assertSetEqual(set(self.input_vars_excluding_protected), set(test_str.split(self._SEP))) - - - class TestSuiteExcludeProtectedCommandLine(TestSuiteCommandLine): - def test_required_variables_excluding_protected(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--exclude-protected", "--required-variables", self.suite_name], - capture_output=True, - text=True) - actualOutput = {s.strip() for s in completedProcess.stdout.split(self._SEP)} - self.assertSetEqual(set(self.required_vars_excluding_protected), actualOutput) - - def test_input_variables_excluding_protected(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--exclude-protected", "--input-variables", self.suite_name], - capture_output=True, - text=True) - actualOutput = {s.strip() for s in completedProcess.stdout.split(self._SEP)} - self.assertSetEqual(set(self.input_vars_excluding_protected), actualOutput) diff --git a/test/unit_tests/README.md b/test/unit_tests/README.md deleted file mode 100644 index 5a2353ee..00000000 --- a/test/unit_tests/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# How to run the python unit-tests - -## Quick start: -To run all unit-tests: -```bash -$ export PYTHONPATH=/scripts:/scripts/parse_tools -$ pytest -v test/ -``` - -To run a specific unit tests: -```bash -$ cd /test/unit_tests -$ python test_metadata_table.py -``` -For more verbose output: -```bash -$ python test_metadata_table.py -v -``` -If you have `coverage` installed, to get test coverage: -```bash -$ coverage run test_metadata_table.py -$ coverage report -m -``` -To check source code quality with pylint: -```bash -$ cd -$ env PYTHONPATH=scripts:${PYTHONPATH} pylint --rcfile ./test/.pylintrc ./test/unit_tests/test_metadata_table.py -$ env PYTHONPATH=scripts:${PYTHONPATH} pylint --rcfile ./test/.pylintrc ./test/unit_tests/test_metadata_scheme_file.py -``` diff --git a/test/unit_tests/sample_files/bad_kind_spec_table_properties.meta b/test/unit_tests/sample_files/bad_kind_spec_table_properties.meta deleted file mode 100644 index d51a30d4..00000000 --- a/test/unit_tests/sample_files/bad_kind_spec_table_properties.meta +++ /dev/null @@ -1,5 +0,0 @@ -[ccpp-table-properties] - name = bad_scheme - type = scheme - dependencies = - kind_spec = temp_r8 diff --git a/test/unit_tests/sample_files/check_fortran_to_metadata.meta b/test/unit_tests/sample_files/check_fortran_to_metadata.meta deleted file mode 100644 index 7d84546b..00000000 --- a/test/unit_tests/sample_files/check_fortran_to_metadata.meta +++ /dev/null @@ -1,31 +0,0 @@ -[ccpp-table-properties] - name = do_stuff - type = scheme - -[ccpp-arg-table] - name = do_stuff_run - type = scheme -[ const_props ] - standard_name = enter_standard_name_1 - units = enter_units - type = ccpp_constituent_prop_ptr_t - dimensions = (enter_standard_name_5:enter_standard_name_6) - intent = in -[ twilight_zone ] - standard_name = enter_standard_name_2 - units = enter_units - type = serling_t - dimensions = () - intent = inout -[ errmsg ] - standard_name = enter_standard_name_3 - units = enter_units - type = character | kind = len=512 - dimensions = () - intent = out -[ errflg ] - standard_name = enter_standard_name_4 - units = enter_units - type = integer - dimensions = () - intent = out diff --git a/test/unit_tests/sample_files/double_header.meta b/test/unit_tests/sample_files/double_header.meta deleted file mode 100644 index 27051c17..00000000 --- a/test/unit_tests/sample_files/double_header.meta +++ /dev/null @@ -1,12 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = host - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host - type = host -[ccpp-arg-table] - name = test_host - type = host diff --git a/test/unit_tests/sample_files/double_table_properties.meta b/test/unit_tests/sample_files/double_table_properties.meta deleted file mode 100644 index 28637b36..00000000 --- a/test/unit_tests/sample_files/double_table_properties.meta +++ /dev/null @@ -1,13 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = host - dependencies = -[ccpp-table-properties] - name = test_host - type = host - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host - type = host diff --git a/test/unit_tests/sample_files/duplicate_kind_spec_table_properties.meta b/test/unit_tests/sample_files/duplicate_kind_spec_table_properties.meta deleted file mode 100644 index 12c1cfc5..00000000 --- a/test/unit_tests/sample_files/duplicate_kind_spec_table_properties.meta +++ /dev/null @@ -1,6 +0,0 @@ -[ccpp-table-properties] - name = scheme_scheme - type = scheme - dependencies = fmodule.F90 - kind_spec = fmodule:kind_temp=>temp_r8 - kind_spec = fmodule:kind_temp diff --git a/test/unit_tests/sample_files/fortran_files/array_parsing_test.F90 b/test/unit_tests/sample_files/fortran_files/array_parsing_test.F90 deleted file mode 100644 index 493ab43d..00000000 --- a/test/unit_tests/sample_files/fortran_files/array_parsing_test.F90 +++ /dev/null @@ -1,33 +0,0 @@ -!Test array specifications -! - -MODULE array_spec_test - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: array_spec_test_run - -CONTAINS - - !> \section arg_table_array_spec_test_run Argument Table - !! \htmlinclude arg_table_array_spec_test_run.html - !! - SUBROUTINE array_spec_test_run(ncol, lev, good_arr1, good_arr2, good_arr3, & - good_arr4, good_arr5, bad_arr1, bad_arr2, bad_arr3) - - integer, intent(in) :: ncol, lev - real(kind_phys), intent(in) :: good_arr1(ncol,lev) - real(kind_phys), intent(in) :: good_arr2(:,:) - real(kind_phys), intent(in), dimension(ncol,lev) :: good_arr3 - real(kind_phys), intent(in), dimension(:,:) :: good_arr4 - real(kind_phys), intent(in) :: good_arr5(ncol,:) - real(kind_phys), intent(in) :: bad_arr1(:,;) - real(kind_phys), intent(in), dimension(;,:) :: bad_arr2 - real(kind_phys), intent(in), dimension(:,;) :: bad_arr3 - - END SUBROUTINE array_spec_test_run - -END MODULE array_spec_test diff --git a/test/unit_tests/sample_files/fortran_files/comments_test.F90 b/test/unit_tests/sample_files/fortran_files/comments_test.F90 deleted file mode 100644 index e74410e7..00000000 --- a/test/unit_tests/sample_files/fortran_files/comments_test.F90 +++ /dev/null @@ -1,34 +0,0 @@ -! -! This work (Common Community Physics Package Framework), identified by -! NOAA, NCAR, CU/CIRES, is free of known copyright restrictions and is -! placed in the public domain. -! -! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -! THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -!> -!! @brief Auto-generated Test of comment writing for FortranWriter -!! -! -module comments_test - -! codee format off -! We can write comments in the module header -! codee format on - ! We can write indented comments in the header - integer :: foo ! Comment at end of line works - integer :: bar ! - ! xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - ! - integer :: baz ! - ! yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy - ! yyyyyyyyyyyyyy - -contains - ! We can write comments in the module body - -end module comments_test diff --git a/test/unit_tests/sample_files/fortran_files/linebreak_test.F90 b/test/unit_tests/sample_files/fortran_files/linebreak_test.F90 deleted file mode 100644 index 2b8f0b74..00000000 --- a/test/unit_tests/sample_files/fortran_files/linebreak_test.F90 +++ /dev/null @@ -1,52 +0,0 @@ -! -! This work (Common Community Physics Package Framework), identified by -! NOAA, NCAR, CU/CIRES, is free of known copyright restrictions and is -! placed in the public domain. -! -! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -! THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -!> -!! @brief Auto-generated Test of line breaking for FortranWriter -!! -! -module linebreak_test - - character(len=7) :: data(100) = (/ 'name000', 'name001', 'name002', 'name003', 'name004', & - 'name005', 'name006', 'name007', 'name008', 'name009', 'name010', 'name011', 'name012', & - 'name013', 'name014', 'name015', 'name016', 'name017', 'name018', 'name019', 'name020', & - 'name021', 'name022', 'name023', 'name024', 'name025', 'name026', 'name027', 'name028', & - 'name029', 'name030', 'name031', 'name032', 'name033', 'name034', 'name035', 'name036', & - 'name037', 'name038', 'name039', 'name040', 'name041', 'name042', 'name043', 'name044', & - 'name045', 'name046', 'name047', 'name048', 'name049', 'name050', 'name051', 'name052', & - 'name053', 'name054', 'name055', 'name056', 'name057', 'name058', 'name059', 'name060', & - 'name061', 'name062', 'name063', 'name064', 'name065', 'name066', 'name067', 'name068', & - 'name069', 'name070', 'name071', 'name072', 'name073', 'name074', 'name075', 'name076', & - 'name077', 'name078', 'name079', 'name080', 'name081', 'name082', 'name083', 'name084', & - 'name085', 'name086', 'name087', 'name088', 'name089', 'name090', 'name091', 'name092', & - 'name093', 'name094', 'name095', 'name096', 'name097', 'name098', 'name099' /) - -contains - - subroutine foo(ozone_constituents, aerosol_constituents, volcaero_constituents, & - other_constituents) - integer, intent(in) :: ozone_constituents(:) - integer, intent(in) :: aerosol_constituents(:) - integer, intent(in) :: volcaero_constituents(:) - integer, intent(in) :: other_constituents(:) - real, allocatable :: tracer_data_test_dynamic_constituents(:) -! codee format off - allocate(tracer_data_test_dynamic_constituents(0+size(ozone_constituents)+size( & - aerosol_constituents)+size(volcaero_constituents)+size(other_constituents))) - - write(6, '(a)') & - 'Cannot read columns_on_task from file'// & - ', columns_on_task has no horizontal dimension; columns_on_task is a protected variable' -! codee format on - end subroutine foo - -end module linebreak_test diff --git a/test/unit_tests/sample_files/fortran_files/long_string_test.F90 b/test/unit_tests/sample_files/fortran_files/long_string_test.F90 deleted file mode 100644 index 17fd17fd..00000000 --- a/test/unit_tests/sample_files/fortran_files/long_string_test.F90 +++ /dev/null @@ -1,97 +0,0 @@ -! -! This work (Common Community Physics Package Framework), identified by -! NOAA, NCAR, CU/CIRES, is free of known copyright restrictions and is -! placed in the public domain. -! -! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -! THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -!> -!! @brief Auto-generated Test of long string breaking for FortranWriter -!! -! -module long_string_test - - character(len=100) :: foo100 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789' - character(len=101) :: foo101 = & - '01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890' - character(len=102) :: foo102 = & - '012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901' - character(len=103) :: foo103 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012' - character(len=104) :: foo104 = & - '01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123' - character(len=105) :: foo105 = & - '012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234' - character(len=106) :: foo106 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345' - character(len=107) :: foo107 = & - '01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456' - character(len=108) :: foo108 = & - '012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567' - character(len=109) :: foo109 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678' - character(len=110) :: foo110 = & - '01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789' - character(len=111) :: foo111 = & - '012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890' - character(len=112) :: foo112 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901' - character(len=113) :: foo113 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &2' - character(len=114) :: foo114 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &23' - character(len=115) :: foo115 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &234' - character(len=116) :: foo116 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &2345' - character(len=117) :: foo117 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &23456' - character(len=118) :: foo118 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &234567' - character(len=119) :: foo119 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &2345678' - character(len=120) :: foo120 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &23456789' - character(len=121) :: foo121 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &234567890' - character(len=122) :: foo122 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &2345678901' - character(len=123) :: foo123 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &23456789012' - character(len=124) :: foo124 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &234567890123' - character(len=125) :: foo125 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &2345678901234' - character(len=126) :: foo126 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &23456789012345' - character(len=127) :: foo127 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &234567890123456' - character(len=128) :: foo128 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &2345678901234567' - character(len=129) :: foo129 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &23456789012345678' - -end module long_string_test diff --git a/test/unit_tests/sample_files/good_kind_spec_table_properties.meta b/test/unit_tests/sample_files/good_kind_spec_table_properties.meta deleted file mode 100644 index 67aeef86..00000000 --- a/test/unit_tests/sample_files/good_kind_spec_table_properties.meta +++ /dev/null @@ -1,6 +0,0 @@ -[ccpp-table-properties] - name = good_scheme - type = scheme - dependencies = fmodule.F90 - kind_spec = fmodule:kind_temp=>temp_r8 - kind_spec = fmodule:temp_i8 diff --git a/test/unit_tests/sample_files/missing_table_properties.meta b/test/unit_tests/sample_files/missing_table_properties.meta deleted file mode 100644 index 0bef09dc..00000000 --- a/test/unit_tests/sample_files/missing_table_properties.meta +++ /dev/null @@ -1,3 +0,0 @@ -[ccpp-arg-table] - name = test_host - type = host diff --git a/test/unit_tests/sample_files/test_bad_1st_arg_table_header.meta b/test/unit_tests/sample_files/test_bad_1st_arg_table_header.meta deleted file mode 100644 index ac6468ac..00000000 --- a/test/unit_tests/sample_files/test_bad_1st_arg_table_header.meta +++ /dev/null @@ -1,23 +0,0 @@ -[ccpp-table-properties] - name = vmr_type - type = ddt - dependencies = - -######################################################################## -[ccpp-farg-table] - name = vmr_type - type = ddt -[ nvmr ] - standard_name = number_of_chemical_species - units = count - dimensions = () - type = integer -[ccpp-arg-table] - name = make_ddt_run - type = scheme -[ nbox ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in diff --git a/test/unit_tests/sample_files/test_bad_2nd_arg_table_header.meta b/test/unit_tests/sample_files/test_bad_2nd_arg_table_header.meta deleted file mode 100644 index 48381744..00000000 --- a/test/unit_tests/sample_files/test_bad_2nd_arg_table_header.meta +++ /dev/null @@ -1,23 +0,0 @@ -[ccpp-table-properties] - name = vmr_type - type = ddt - dependencies = - -######################################################################## -[ccpp-arg-table] - name = vmr_type - type = ddt -[ nvmr ] - standard_name = number_of_chemical_species - units = count - dimensions = () - type = integer -[ccpp-farg-table] - name = make_ddt_run - type = scheme -[ nbox ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in diff --git a/test/unit_tests/sample_files/test_bad_dimension.meta b/test/unit_tests/sample_files/test_bad_dimension.meta deleted file mode 100644 index 9a20b1f8..00000000 --- a/test/unit_tests/sample_files/test_bad_dimension.meta +++ /dev/null @@ -1,15 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = host - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host - type = host -[ col_start ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = banana - protected = True diff --git a/test/unit_tests/sample_files/test_bad_line_split.meta b/test/unit_tests/sample_files/test_bad_line_split.meta deleted file mode 100644 index 3ace2ccb..00000000 --- a/test/unit_tests/sample_files/test_bad_line_split.meta +++ /dev/null @@ -1,16 +0,0 @@ -[ccpp-table-properties] - name = temp_calc_adjust - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = temp_calc_adjust_run - type = scheme -[ temp_calc ] - standard_name = potential_temperature_at_previous_timestep - units = K | - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = out diff --git a/test/unit_tests/sample_files/test_bad_table_key.meta b/test/unit_tests/sample_files/test_bad_table_key.meta deleted file mode 100644 index 8d0bbc7f..00000000 --- a/test/unit_tests/sample_files/test_bad_table_key.meta +++ /dev/null @@ -1,10 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = host - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host - type = host - banana = something diff --git a/test/unit_tests/sample_files/test_bad_table_type.meta b/test/unit_tests/sample_files/test_bad_table_type.meta deleted file mode 100644 index b6b9449d..00000000 --- a/test/unit_tests/sample_files/test_bad_table_type.meta +++ /dev/null @@ -1,15 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host - type = host -[ col_start ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = () - protected = True diff --git a/test/unit_tests/sample_files/test_bad_type_name.meta b/test/unit_tests/sample_files/test_bad_type_name.meta deleted file mode 100644 index de11315d..00000000 --- a/test/unit_tests/sample_files/test_bad_type_name.meta +++ /dev/null @@ -1,9 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host - type = banana diff --git a/test/unit_tests/sample_files/test_bad_var_property_name.meta b/test/unit_tests/sample_files/test_bad_var_property_name.meta deleted file mode 100644 index a2009233..00000000 --- a/test/unit_tests/sample_files/test_bad_var_property_name.meta +++ /dev/null @@ -1,35 +0,0 @@ -[ccpp-table-properties] - name = vmr_type - type = ddt - dependencies = - -######################################################################## -[ccpp-arg-table] - name = vmr_type - type = ddt -[ nvmr ] - standard_name = number_of_chemical_species - units = count - dimensions = () - type = integer -[ vmr_array ] - standard_name = array_of_volume_mixing_ratios - units = ppmv - dimensions = (horizontal_dimension, number_of_chemical_species) - type = real - kind = kind_phys - -[ccpp-table-properties] - name = make_ddt - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = make_ddt_run - type = scheme -[ vmr ] - standard_name = volume_mixing_ratio_ddt - dimensions = () - None = None - intent = inout diff --git a/test/unit_tests/sample_files/test_dependencies_path.meta b/test/unit_tests/sample_files/test_dependencies_path.meta deleted file mode 100644 index 391ad93d..00000000 --- a/test/unit_tests/sample_files/test_dependencies_path.meta +++ /dev/null @@ -1,11 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = host - dependencies_path = ../../ccpp/physics/physics - dependencies = machine.F,physcons.F90,, - dependencies = GFDL_parse_tracers.F90,,rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90 - -######################################################################## -[ccpp-arg-table] - name = test_host - type = host diff --git a/test/unit_tests/sample_files/test_duplicate_variable.meta b/test/unit_tests/sample_files/test_duplicate_variable.meta deleted file mode 100644 index efc66b86..00000000 --- a/test/unit_tests/sample_files/test_duplicate_variable.meta +++ /dev/null @@ -1,21 +0,0 @@ -[ccpp-table-properties] - name = temp_calc_adjust - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = temp_calc_adjust_run - type = scheme -[ temp ] - standard_name = index_of_water_vapor_specific_humidity - units = index - type = integer - intent = in - dimensions = () -[ temp ] - standard_name = index_of_water_vapor_specific_humidity - units = index - type = integer - intent = in - dimensions = () diff --git a/test/unit_tests/sample_files/test_fortran_to_metadata.F90 b/test/unit_tests/sample_files/test_fortran_to_metadata.F90 deleted file mode 100644 index ff4542c4..00000000 --- a/test/unit_tests/sample_files/test_fortran_to_metadata.F90 +++ /dev/null @@ -1,28 +0,0 @@ -module dme_adjust - - use ccpp_kinds, only: kind_phys - - implicit none - -contains -!=============================================================================== -!> \section arg_table_do_stuff_run Argument Table -!! \htmlinclude do_stuff_run.html -!! - subroutine do_stuff_run(const_props, twilight_zone, errmsg, errflg) - ! - ! Arguments - ! - type(ccpp_constituent_prop_ptr_t), intent(in) :: const_props(:) - type(serling_t), intent(inout) :: twilight_zone - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - errmsg = ' ' - errflg = 0 - twilight_zone('adjust_set') - - end subroutine dme_adjust_run - -end module dme_adjust diff --git a/test/unit_tests/sample_files/test_host.meta b/test/unit_tests/sample_files/test_host.meta deleted file mode 100644 index f618f871..00000000 --- a/test/unit_tests/sample_files/test_host.meta +++ /dev/null @@ -1,34 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = host - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host - type = host -[ col_start ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = () - protected = True -[ col_end ] - standard_name = horizontal_loop_end - type = integer - units = count - dimensions = () - protected = True -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer diff --git a/test/unit_tests/sample_files/test_invalid_intent.meta b/test/unit_tests/sample_files/test_invalid_intent.meta deleted file mode 100644 index 6f11169e..00000000 --- a/test/unit_tests/sample_files/test_invalid_intent.meta +++ /dev/null @@ -1,23 +0,0 @@ -[ccpp-table-properties] - name = temp_calc_adjust - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = temp_calc_adjust_run - type = scheme -[ nbox ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = banana diff --git a/test/unit_tests/sample_files/test_invalid_table_properties_type.meta b/test/unit_tests/sample_files/test_invalid_table_properties_type.meta deleted file mode 100644 index 45f43e83..00000000 --- a/test/unit_tests/sample_files/test_invalid_table_properties_type.meta +++ /dev/null @@ -1,9 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = banana - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host - type = host diff --git a/test/unit_tests/sample_files/test_mismatch_section_table_title.meta b/test/unit_tests/sample_files/test_mismatch_section_table_title.meta deleted file mode 100644 index 240b93c2..00000000 --- a/test/unit_tests/sample_files/test_mismatch_section_table_title.meta +++ /dev/null @@ -1,9 +0,0 @@ -[ccpp-table-properties] - name = banana - type = host - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host - type = host diff --git a/test/unit_tests/sample_files/test_missing_intent.meta b/test/unit_tests/sample_files/test_missing_intent.meta deleted file mode 100644 index 57be3ad6..00000000 --- a/test/unit_tests/sample_files/test_missing_intent.meta +++ /dev/null @@ -1,22 +0,0 @@ -[ccpp-table-properties] - name = temp_calc_adjust - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = temp_calc_adjust_run - type = scheme -[ nbox ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys diff --git a/test/unit_tests/sample_files/test_missing_table_name.meta b/test/unit_tests/sample_files/test_missing_table_name.meta deleted file mode 100644 index b8deb22c..00000000 --- a/test/unit_tests/sample_files/test_missing_table_name.meta +++ /dev/null @@ -1,14 +0,0 @@ -[ccpp-table-properties] - name = test_missing_table_name - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - type = host -[ col_start ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = () - protected = True diff --git a/test/unit_tests/sample_files/test_missing_table_type.meta b/test/unit_tests/sample_files/test_missing_table_type.meta deleted file mode 100644 index 98b5bd4f..00000000 --- a/test/unit_tests/sample_files/test_missing_table_type.meta +++ /dev/null @@ -1,14 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host -[ col_start ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = () - protected = True diff --git a/test/unit_tests/sample_files/test_missing_units.meta b/test/unit_tests/sample_files/test_missing_units.meta deleted file mode 100644 index 1b9546f2..00000000 --- a/test/unit_tests/sample_files/test_missing_units.meta +++ /dev/null @@ -1,16 +0,0 @@ -[ccpp-table-properties] - name = temp_calc_adjust - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = temp_calc_adjust_run - type = scheme -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - dimensions = () - type = real - kind = kind_phys - intent = in diff --git a/test/unit_tests/sample_files/test_multi_ccpp_arg_tables.meta b/test/unit_tests/sample_files/test_multi_ccpp_arg_tables.meta deleted file mode 100644 index 0be34179..00000000 --- a/test/unit_tests/sample_files/test_multi_ccpp_arg_tables.meta +++ /dev/null @@ -1,115 +0,0 @@ -[ccpp-table-properties] - name = vmr_type - type = ddt - dependencies = - -[ccpp-arg-table] - name = vmr_type - type = ddt -[ nvmr ] - standard_name = number_of_chemical_species - units = count - dimensions = () - type = integer -[ vmr_array ] - standard_name = array_of_volume_mixing_ratios - units = ppmv - dimensions = (horizontal_dimension, number_of_chemical_species) - type = real - kind = kind_phys - -######################################################################## -[ccpp-table-properties] - name = make_ddt - type = scheme - dependencies = - -[ccpp-arg-table] - name = make_ddt_run - type = scheme -[ nbox ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ O3 ] - standard_name = ozone - units = ppmv - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[ HNO3 ] - standard_name = nitric_acid - units = ppmv - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[ vmr ] - standard_name = volume_mixing_ratio_ddt - dimensions = () - type = vmr_type - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none | dimensions = () | type = character | kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out - -[ccpp-arg-table] - name = make_ddt_init - type = scheme -[ nbox ] - standard_name = horizontal_dimension - type = integer - units = count - dimensions = () - intent = in -[ vmr ] - standard_name = volume_mixing_ratio_ddt - dimensions = () - type = vmr_type - intent = out -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out - -[ccpp-arg-table] - name = make_ddt_finalize - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_files/test_unknown_ddt_type.meta b/test/unit_tests/sample_files/test_unknown_ddt_type.meta deleted file mode 100644 index e41eeefb..00000000 --- a/test/unit_tests/sample_files/test_unknown_ddt_type.meta +++ /dev/null @@ -1,14 +0,0 @@ -[ccpp-table-properties] - name = make_ddt - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = make_ddt_run - type = scheme -[ vmr ] - standard_name = volume_mixing_ratio_ddt - dimensions = () - type = banana - intent = inout diff --git a/test/unit_tests/sample_host_files/data1_mod.F90 b/test/unit_tests/sample_host_files/data1_mod.F90 deleted file mode 100644 index b85db315..00000000 --- a/test/unit_tests/sample_host_files/data1_mod.F90 +++ /dev/null @@ -1,11 +0,0 @@ -module data1_mod - - use ccpp_kinds, only: kind_phys - - !> \section arg_table_data1_mod Argument Table - !! \htmlinclude arg_table_data1_mod.html - real(kind_phys) :: ps1 - real(kind_phys), allocatable :: xbox(:,:) - real(kind_phys), allocatable :: switch(:,:) - -end module data1_mod diff --git a/test/unit_tests/sample_host_files/data1_mod.meta b/test/unit_tests/sample_host_files/data1_mod.meta deleted file mode 100644 index 37e2de96..00000000 --- a/test/unit_tests/sample_host_files/data1_mod.meta +++ /dev/null @@ -1,25 +0,0 @@ -[ccpp-table-properties] - name = data1_mod - type = module -[ccpp-arg-table] - name = data1_mod - type = module -[ ps1 ] - standard_name = play_station - state_variable = true - type = real | kind = kind_phys - units = Pa - dimensions = () -[ xbox ] - standard_name = xbox - state_variable = true - type = real | kind = kind_phys - units = m s-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) -[ switch ] - standard_name = nintendo_switch - long_name = Incompatible junk - state_variable = true - type = real | kind = kind_phys - units = m s-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) diff --git a/test/unit_tests/sample_host_files/ddt1.F90 b/test/unit_tests/sample_host_files/ddt1.F90 deleted file mode 100644 index 71b22b4f..00000000 --- a/test/unit_tests/sample_host_files/ddt1.F90 +++ /dev/null @@ -1,17 +0,0 @@ -module ddt1 - - use ccpp_kinds, only: kind_phys - - private - implicit none - - !! \section arg_table_ddt1_t - !! \htmlinclude ddt1_t.html - !! - type, public :: ddt1_t - integer, public :: num_vars = 0 - real(kind_phys), allocatable :: vars(:,:,:) - - end type ddt1_t - -end module ddt1 diff --git a/test/unit_tests/sample_host_files/ddt1.meta b/test/unit_tests/sample_host_files/ddt1.meta deleted file mode 100644 index e1a0f1ac..00000000 --- a/test/unit_tests/sample_host_files/ddt1.meta +++ /dev/null @@ -1,20 +0,0 @@ -######################################################################## -[ccpp-table-properties] - name = ddt1_t - type = ddt - -[ccpp-arg-table] - name = ddt1_t - type = ddt -[ num_vars ] - standard_name = ddt_var_array_dimension - long_name = Number of vars managed by ddt1 - units = count - dimensions = () - type = integer -[ vars ] - standard_name = vars_array - long_name = Array of vars managed by ddt1 - units = none - dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) - type = real | kind = kind_phys diff --git a/test/unit_tests/sample_host_files/ddt1_plus.F90 b/test/unit_tests/sample_host_files/ddt1_plus.F90 deleted file mode 100644 index d1806932..00000000 --- a/test/unit_tests/sample_host_files/ddt1_plus.F90 +++ /dev/null @@ -1,33 +0,0 @@ -module ddt1_plus - - use ccpp_kinds, only: kind_phys - - private - implicit none - - !! - type, public :: ddt1_t - real, pointer :: undocumented_array(:) => NULL() - contains - procedure :: this_is_a_documented_object - end type ddt1_t - - !! \section arg_table_ddt2_t - !! \htmlinclude ddt2_t.html - !! - type, public :: ddt2_t - integer, public :: num_vars = 0 - real(kind_phys), allocatable :: vars(:,:,:) - - end type ddt2_t - -CONTAINS - - logical function this_is_a_documented_object(this) - class(ddt1_t) :: intent(in) :: this - - this_is_a_documented_object = .false. - - end function this_is_a_documented_object - -end module ddt1_plus diff --git a/test/unit_tests/sample_host_files/ddt1_plus.meta b/test/unit_tests/sample_host_files/ddt1_plus.meta deleted file mode 100644 index ca3a92ab..00000000 --- a/test/unit_tests/sample_host_files/ddt1_plus.meta +++ /dev/null @@ -1,20 +0,0 @@ -######################################################################## -[ccpp-table-properties] - name = ddt2_t - type = ddt - -[ccpp-arg-table] - name = ddt2_t - type = ddt -[ num_vars ] - standard_name = ddt_var_array_dimension - long_name = Number of vars managed by ddt2 - units = count - dimensions = () - type = integer -[ vars ] - standard_name = vars_array - long_name = Array of vars managed by ddt2 - units = none - dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) - type = real | kind = kind_phys diff --git a/test/unit_tests/sample_host_files/ddt2.F90 b/test/unit_tests/sample_host_files/ddt2.F90 deleted file mode 100644 index 22d5af0e..00000000 --- a/test/unit_tests/sample_host_files/ddt2.F90 +++ /dev/null @@ -1,24 +0,0 @@ -module ddt2 - - use ccpp_kinds, only: kind_phys - - private - implicit none - - !! \section arg_table_ddt1_t - !! \htmlinclude ddt1_t.html - !! - type, public :: ddt1_t - real, pointer :: undocumented_array(:) => NULL() - end type ddt1_t - - !! \section arg_table_ddt2_t - !! \htmlinclude ddt2_t.html - !! - type, public :: ddt2_t - integer, public :: num_vars = 0 - real(kind_phys), allocatable :: vars(:,:,:) - - end type ddt2_t - -end module ddt2 diff --git a/test/unit_tests/sample_host_files/ddt2.meta b/test/unit_tests/sample_host_files/ddt2.meta deleted file mode 100644 index 159f08b0..00000000 --- a/test/unit_tests/sample_host_files/ddt2.meta +++ /dev/null @@ -1,29 +0,0 @@ -######################################################################## -[ccpp-table-properties] - name = ddt1_t - type = ddt - -[ccpp-arg-table] - name = ddt1_t - type = ddt - -######################################################################## -[ccpp-table-properties] - name = ddt2_t - type = ddt - -[ccpp-arg-table] - name = ddt2_t - type = ddt -[ num_vars ] - standard_name = ddt_var_array_dimension - long_name = Number of vars managed by ddt2 - units = count - dimensions = () - type = integer -[ vars ] - standard_name = vars_array - long_name = Array of vars managed by ddt2 - units = none - dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) - type = real | kind = kind_phys diff --git a/test/unit_tests/sample_host_files/ddt2_extra_var.F90 b/test/unit_tests/sample_host_files/ddt2_extra_var.F90 deleted file mode 100644 index 00b4c170..00000000 --- a/test/unit_tests/sample_host_files/ddt2_extra_var.F90 +++ /dev/null @@ -1,34 +0,0 @@ -module ddt2_extra_var - - use ccpp_kinds, only: kind_phys - - private - implicit none - - !! \section arg_table_ddt1_t - !! \htmlinclude ddt1_t.html - !! - type, public :: ddt1_t - real, pointer :: undocumented_array(:) => NULL() - end type ddt1_t - - !! \section arg_table_ddt2_t - !! \htmlinclude ddt2_t.html - !! - type, public :: ddt2_t - integer, public :: num_vars = 0 - real(kind_phys), allocatable :: vars(:,:,:) - contains - procedure :: get_num_vars - end type ddt2_t - -CONTAINS - - integer function get_num_vars(this) - class(ddt2_t), intent(in) :: this - - get_num_vars = this%num_vars - - end function get_num_vars - -end module ddt2_extra_var diff --git a/test/unit_tests/sample_host_files/ddt2_extra_var.meta b/test/unit_tests/sample_host_files/ddt2_extra_var.meta deleted file mode 100644 index 867720e5..00000000 --- a/test/unit_tests/sample_host_files/ddt2_extra_var.meta +++ /dev/null @@ -1,34 +0,0 @@ -######################################################################## -[ccpp-table-properties] - name = ddt1_t - type = ddt - -[ccpp-arg-table] - name = ddt1_t - type = ddt - -######################################################################## -[ccpp-table-properties] - name = ddt2_t - type = ddt - -[ccpp-arg-table] - name = ddt2_t - type = ddt -[ num_vars ] - standard_name = ddt_var_array_dimension - long_name = Number of vars managed by ddt2 - units = count - dimensions = () - type = integer -[ vars ] - standard_name = vars_array - long_name = Array of vars managed by ddt2 - units = none - dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) - type = real | kind = kind_phys -[ bogus ] - standard_name = misplaced_variable - units = count - dimensions = () - type = integer diff --git a/test/unit_tests/sample_host_files/ddt_data1_mod.F90 b/test/unit_tests/sample_host_files/ddt_data1_mod.F90 deleted file mode 100644 index 5efe0845..00000000 --- a/test/unit_tests/sample_host_files/ddt_data1_mod.F90 +++ /dev/null @@ -1,30 +0,0 @@ -module ddt_data1_mod - - use ccpp_kinds, only: kind_phys - - private - implicit none - - !! \section arg_table_ddt1_t - !! \htmlinclude ddt1_t.html - !! - type, public :: ddt1_t - real, pointer :: undocumented_array(:) => NULL() - end type ddt1_t - - !! \section arg_table_ddt2_t - !! \htmlinclude ddt2_t.html - !! - type, public :: ddt2_t - integer, public :: num_vars = 0 - real(kind_phys), allocatable :: vars(:,:,:) - - end type ddt2_t - - !> \section arg_table_ddt_data1_mod Argument Table - !! \htmlinclude arg_table_ddt_data1_mod.html - real(kind_phys) :: ps1 - real(kind_phys), allocatable :: xbox(:,:) - real(kind_phys), allocatable :: switch(:,:) - -end module ddt_data1_mod diff --git a/test/unit_tests/sample_host_files/ddt_data1_mod.meta b/test/unit_tests/sample_host_files/ddt_data1_mod.meta deleted file mode 100644 index e149c07b..00000000 --- a/test/unit_tests/sample_host_files/ddt_data1_mod.meta +++ /dev/null @@ -1,56 +0,0 @@ -######################################################################## -[ccpp-table-properties] - name = ddt1_t - type = ddt - -[ccpp-arg-table] - name = ddt1_t - type = ddt - -######################################################################## -[ccpp-table-properties] - name = ddt2_t - type = ddt - -[ccpp-arg-table] - name = ddt2_t - type = ddt -[ num_vars ] - standard_name = ddt_var_array_dimension - long_name = Number of vars managed by ddt2 - units = count - dimensions = () - type = integer -[ vars ] - standard_name = vars_array - long_name = Array of vars managed by ddt2 - units = none - dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) - type = real | kind = kind_phys - -######################################################################## -[ccpp-table-properties] - name = ddt_data1_mod - type = module -[ccpp-arg-table] - name = ddt_data1_mod - type = module -[ ps1 ] - standard_name = play_station - state_variable = true - type = real | kind = kind_phys - units = Pa - dimensions = () -[ xbox ] - standard_name = xbox - state_variable = true - type = real | kind = kind_phys - units = m s-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) -[ switch ] - standard_name = nintendo_switch - long_name = Incompatible junk - state_variable = true - type = real | kind = kind_phys - units = m s-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) diff --git a/test/unit_tests/sample_host_files/mismatch_hdim_mod.F90 b/test/unit_tests/sample_host_files/mismatch_hdim_mod.F90 deleted file mode 100644 index b3ebe52b..00000000 --- a/test/unit_tests/sample_host_files/mismatch_hdim_mod.F90 +++ /dev/null @@ -1,11 +0,0 @@ -module mismatch_hdim_mod - - use ccpp_kinds, only: kind_phys - - !> \section arg_table_mismatch_hdim_mod Argument Table - !! \htmlinclude arg_table_mismatch_hdim_mod.html - real(kind_phys) :: ps1 - real(kind_phys), allocatable :: xbox(:,:) - real(kind_phys), allocatable :: switch(:,:) - -end module mismatch_hdim_mod diff --git a/test/unit_tests/sample_host_files/mismatch_hdim_mod.meta b/test/unit_tests/sample_host_files/mismatch_hdim_mod.meta deleted file mode 100644 index 24f6ba77..00000000 --- a/test/unit_tests/sample_host_files/mismatch_hdim_mod.meta +++ /dev/null @@ -1,25 +0,0 @@ -[ccpp-table-properties] - name = mismatch_hdim_mod - type = module -[ccpp-arg-table] - name = mismatch_hdim_mod - type = module -[ ps1 ] - standard_name = play_station - state_variable = true - type = real | kind = kind_phys - units = Pa - dimensions = () -[ xbox ] - standard_name = xbox - state_variable = true - type = real | kind = kind_phys - units = m s-1 - dimensions = (horizontal_loop_extent, vertical_layer_dimension) -[ switch ] - standard_name = nintendo_switch - long_name = Incompatible junk - state_variable = true - type = real | kind = kind_phys - units = m s-1 - dimensions = (horizontal_loop_being:horizontal_loop_end, vertical_layer_dimension) diff --git a/test/unit_tests/sample_scheme_files/CCPPeq1_var_in_fort_meta.F90 b/test/unit_tests/sample_scheme_files/CCPPeq1_var_in_fort_meta.F90 deleted file mode 100644 index 20446f64..00000000 --- a/test/unit_tests/sample_scheme_files/CCPPeq1_var_in_fort_meta.F90 +++ /dev/null @@ -1,38 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE CCPPeq1_var_in_fort_meta - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: CCPPeq1_var_in_fort_meta_run - -CONTAINS - - !> \section arg_table_CCPPeq1_var_in_fort_meta_run Argument Table - !! \htmlinclude arg_table_CCPPeq1_var_in_fort_meta_run.html - !! - subroutine CCPPeq1_var_in_fort_meta_run (foo, & -#ifdef CCPP - bar, & -#endif - errmsg, errflg) - - integer, intent(in) :: foo -#ifdef CCPP - real(kind_phys), intent(in) :: bar -#endif - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine CCPPeq1_var_in_fort_meta_run - -END MODULE CCPPeq1_var_in_fort_meta diff --git a/test/unit_tests/sample_scheme_files/CCPPeq1_var_in_fort_meta.meta b/test/unit_tests/sample_scheme_files/CCPPeq1_var_in_fort_meta.meta deleted file mode 100644 index 9a05a3ac..00000000 --- a/test/unit_tests/sample_scheme_files/CCPPeq1_var_in_fort_meta.meta +++ /dev/null @@ -1,37 +0,0 @@ -[ccpp-table-properties] - name = CCPPeq1_var_in_fort_meta - type = scheme - -######################################################################## -[ccpp-arg-table] - name = CCPPeq1_var_in_fort_meta_run - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ bar ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_fort.F90 b/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_fort.F90 deleted file mode 100644 index 85b9b370..00000000 --- a/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_fort.F90 +++ /dev/null @@ -1,38 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE CCPPeq1_var_missing_in_fort - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: CCPPeq1_var_missing_in_fort_run - -CONTAINS - - !> \section arg_table_CCPPeq1_var_missing_in_fort_run Argument Table - !! \htmlinclude arg_table_CCPPeq1_var_missing_in_fort_run.html - !! - subroutine CCPPeq1_var_missing_in_fort_run (foo, & -#ifndef CCPP - bar, & -#endif - errmsg, errflg) - - integer, intent(in) :: foo -#ifndef CCPP - real(kind_phys), intent(in) :: bar -#endif - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine CCPPeq1_var_missing_in_fort_run - -END MODULE CCPPeq1_var_missing_in_fort diff --git a/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_fort.meta b/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_fort.meta deleted file mode 100644 index 283c697d..00000000 --- a/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_fort.meta +++ /dev/null @@ -1,37 +0,0 @@ -[ccpp-table-properties] - name = CCPPeq1_var_missing_in_fort - type = scheme - -######################################################################## -[ccpp-arg-table] - name = CCPPeq1_var_missing_in_fort_run - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ bar ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_meta.F90 b/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_meta.F90 deleted file mode 100644 index 155db942..00000000 --- a/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_meta.F90 +++ /dev/null @@ -1,38 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE CCPPeq1_var_missing_in_meta - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: CCPPeq1_var_missing_in_meta_finalize - -CONTAINS - - !> \section arg_table_CCPPeq1_var_missing_in_meta_finalize Argument Table - !! \htmlinclude arg_table_CCPPeq1_var_missing_in_meta_finalize.html - !! - subroutine CCPPeq1_var_missing_in_meta_finalize (foo, & -#ifdef CCPP - bar, & -#endif - errmsg, errflg) - - integer, intent(in) :: foo -#ifdef CCPP - real(kind_phys), intent(in) :: bar -#endif - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine CCPPeq1_var_missing_in_meta_finalize - -END MODULE CCPPeq1_var_missing_in_meta diff --git a/test/unit_tests/sample_scheme_files/CCPPgt1_var_in_fort_meta.F90 b/test/unit_tests/sample_scheme_files/CCPPgt1_var_in_fort_meta.F90 deleted file mode 100644 index fed23ff0..00000000 --- a/test/unit_tests/sample_scheme_files/CCPPgt1_var_in_fort_meta.F90 +++ /dev/null @@ -1,38 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE CCPPgt1_var_in_fort_meta - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: CCPPgt1_var_in_fort_meta_init - -CONTAINS - - !> \section arg_table_CCPPgt1_var_in_fort_meta_init Argument Table - !! \htmlinclude arg_table_CCPPgt1_var_in_fort_meta_init.html - !! - subroutine CCPPgt1_var_in_fort_meta_init (foo, & -#if CCPP > 1 - bar, & -#endif - errmsg, errflg) - - integer, intent(in) :: foo -#if CCPP > 1 - real(kind_phys), intent(in) :: bar -#endif - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine CCPPgt1_var_in_fort_meta_init - -END MODULE CCPPgt1_var_in_fort_meta diff --git a/test/unit_tests/sample_scheme_files/CCPPgt1_var_in_fort_meta.meta b/test/unit_tests/sample_scheme_files/CCPPgt1_var_in_fort_meta.meta deleted file mode 100644 index 00924ba6..00000000 --- a/test/unit_tests/sample_scheme_files/CCPPgt1_var_in_fort_meta.meta +++ /dev/null @@ -1,37 +0,0 @@ -[ccpp-table-properties] - name = CCPPgt1_var_in_fort_meta - type = scheme - -######################################################################## -[ccpp-arg-table] - name = CCPPgt1_var_in_fort_meta_init - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ bar ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/CCPPnotset_var_missing_in_meta.F90 b/test/unit_tests/sample_scheme_files/CCPPnotset_var_missing_in_meta.F90 deleted file mode 100644 index 14a49168..00000000 --- a/test/unit_tests/sample_scheme_files/CCPPnotset_var_missing_in_meta.F90 +++ /dev/null @@ -1,38 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE CCPPnotset_var_missing_in_meta - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: CCPPnotset_var_missing_in_meta_run - -CONTAINS - - !> \section arg_table_CCPPnotset_var_missing_in_meta_run Argument Table - !! \htmlinclude arg_table_CCPPnotset_var_missing_in_meta_run.html - !! - subroutine CCPPnotset_var_missing_in_meta_run (foo, & -#ifndef CCPP - bar, & -#endif - errmsg, errflg) - - integer, intent(in) :: foo -#ifndef CCPP - real(kind_phys), intent(in) :: bar -#endif - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine CCPPnotset_var_missing_in_meta_run - -END MODULE CCPPnotset_var_missing_in_meta diff --git a/test/unit_tests/sample_scheme_files/CCPPnotset_var_missing_in_meta.meta b/test/unit_tests/sample_scheme_files/CCPPnotset_var_missing_in_meta.meta deleted file mode 100644 index c6435aab..00000000 --- a/test/unit_tests/sample_scheme_files/CCPPnotset_var_missing_in_meta.meta +++ /dev/null @@ -1,29 +0,0 @@ -[ccpp-table-properties] - name = CCPPnotset_var_missing_in_meta - type = scheme - -######################################################################## -[ccpp-arg-table] - name = CCPPnotset_var_missing_in_meta_run - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/invalid_dummy_arg.F90 b/test/unit_tests/sample_scheme_files/invalid_dummy_arg.F90 deleted file mode 100644 index 16f93864..00000000 --- a/test/unit_tests/sample_scheme_files/invalid_dummy_arg.F90 +++ /dev/null @@ -1,43 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE invalid_dummy_arg - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: invalid_dummy_arg_run - -CONTAINS - - !> \section arg_table_invalid_dummy_arg_run Argument Table - !! \htmlinclude arg_table_invalid_dummy_arg_run.html - !! - subroutine invalid_dummy_arg_run(foo, timestep, temp_prev, temp_layer, qv, ps, & - errmsg, errflg) - - integer, intent(in) :: foo - real(kind_phys), intent(in) :: timestep - real(kind_phys), intent(inout) :: qv(:) - real(kind_phys), intent(inout) :: ps(:) - REAL(kind_phys), intent(in) :: woohoo(:) - REAL(kind_phys), intent(inout) :: temp_layer(foo) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: col_index - - errmsg = '' - errflg = 0 - - do col_index = 1, foo - temp_layer(col_index) = temp_layer(col_index) + temp_prev(col_index) - qv(col_index) = qv(col_index) + 1.0_kind_phys - end do - - END SUBROUTINE invalid_dummy_arg_run - -END MODULE invalid_dummy_arg diff --git a/test/unit_tests/sample_scheme_files/invalid_dummy_arg.meta b/test/unit_tests/sample_scheme_files/invalid_dummy_arg.meta deleted file mode 100644 index 4dd8e9e6..00000000 --- a/test/unit_tests/sample_scheme_files/invalid_dummy_arg.meta +++ /dev/null @@ -1,66 +0,0 @@ -[ccpp-table-properties] - name = invalid_dummy_arg - type = scheme - -######################################################################## -[ccpp-arg-table] - name = invalid_dummy_arg_run - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ temp_prev ] - standard_name = potential_temperature_at_previous_timestep - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[ temp_layer ] - standard_name = potential_temperature - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ qv ] - standard_name = water_vapor_specific_humidity - units = kg kg-1 - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ ps ] - standard_name = surface_air_pressure - state_variable = true - type = real - kind = kind_phys - units = Pa - dimensions = (horizontal_loop_extent) - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/invalid_subr_stmnt.F90 b/test/unit_tests/sample_scheme_files/invalid_subr_stmnt.F90 deleted file mode 100644 index 98100553..00000000 --- a/test/unit_tests/sample_scheme_files/invalid_subr_stmnt.F90 +++ /dev/null @@ -1,30 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE invalid_subr_stmnt - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: invalid_subr_stmnt_init - -CONTAINS - - !> \section arg_table_invalid_subr_stmnt_init Argument Table - !! \htmlinclude arg_table_invalid_subr_stmnt_init.html - !! - subroutine invalid_subr_stmnt_init (woohoo, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine invalid_subr_stmnt_init - -END MODULE invalid_subr_stmnt diff --git a/test/unit_tests/sample_scheme_files/mismatch_hdim.F90 b/test/unit_tests/sample_scheme_files/mismatch_hdim.F90 deleted file mode 100644 index 67680917..00000000 --- a/test/unit_tests/sample_scheme_files/mismatch_hdim.F90 +++ /dev/null @@ -1,48 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE mismatch_hdim - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: mismatch_hdim_init - PUBLIC :: mismatch_hdim_run - -CONTAINS - - !> \section arg_table_mismatch_hdim_run Argument Table - !! \htmlinclude arg_table_mismatch_hdim_run.html - !! - subroutine mismatch_hdim_run(tsfc, errmsg, errflg) - - real(kind_phys), intent(inout) :: tsfc(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - errmsg = '' - errflg = 0 - - tsfc = tsfc-1.0_kind_phys - - END SUBROUTINE mismatch_hdim_run - - !> \section arg_table_mismatch_hdim_init Argument Table - !! \htmlinclude arg_table_mismatch_hdim_init.html - !! - subroutine mismatch_hdim_init (tsfc, errmsg, errflg) - - real(kind_phys), intent(inout) :: tsfc(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - tsfc = tsfc+1.0_kind_phys - - errmsg = '' - errflg = 0 - - end subroutine mismatch_hdim_init - -END MODULE mismatch_hdim diff --git a/test/unit_tests/sample_scheme_files/mismatch_hdim.meta b/test/unit_tests/sample_scheme_files/mismatch_hdim.meta deleted file mode 100644 index 55d87fc3..00000000 --- a/test/unit_tests/sample_scheme_files/mismatch_hdim.meta +++ /dev/null @@ -1,55 +0,0 @@ -[ccpp-table-properties] - name = mismatch_hdim - type = scheme - -######################################################################## -[ccpp-arg-table] - name = mismatch_hdim_run - type = scheme -[ tsfc ] - standard_name = temperature_at_surface - units = K - dimensions = (horizontal_dimension) - type = real - kind = kind_phys - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = mismatch_hdim_init - type = scheme -[ tsfc ] - standard_name = temperature_at_surface - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/mismatch_intent.F90 b/test/unit_tests/sample_scheme_files/mismatch_intent.F90 deleted file mode 100644 index abcf7bc0..00000000 --- a/test/unit_tests/sample_scheme_files/mismatch_intent.F90 +++ /dev/null @@ -1,75 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE mismatch_intent - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: mismatch_intent_init - PUBLIC :: mismatch_intent_run - PUBLIC :: mismatch_intent_finalize - -CONTAINS - - !> \section arg_table_mismatch_intent_run Argument Table - !! \htmlinclude arg_table_mismatch_intent_run.html - !! - subroutine mismatch_intent_run(foo, timestep, temp_prev, temp_layer, qv, ps, & - errmsg, errflg) - - integer, intent(in) :: foo - real(kind_phys), intent(in) :: timestep - real(kind_phys), intent(inout) :: qv(:) - real(kind_phys), intent(inout) :: ps(:) - REAL(kind_phys), intent(in) :: temp_prev(:) - REAL(kind_phys), intent(inout) :: temp_layer(foo) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: col_index - - errmsg = '' - errflg = 0 - - do col_index = 1, foo - temp_layer(col_index) = temp_layer(col_index) + temp_prev(col_index) - qv(col_index) = qv(col_index) + 1.0_kind_phys - end do - - END SUBROUTINE mismatch_intent_run - - !> \section arg_table_mismatch_intent_init Argument Table - !! \htmlinclude arg_table_mismatch_intent_init.html - !! - subroutine mismatch_intent_init (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine mismatch_intent_init - - !> \section arg_table_mismatch_intent_finalize Argument Table - !! \htmlinclude arg_table_mismatch_intent_finalize.html - !! - subroutine mismatch_intent_finalize (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine mismatch_intent_finalize - -END MODULE mismatch_intent diff --git a/test/unit_tests/sample_scheme_files/mismatch_intent.meta b/test/unit_tests/sample_scheme_files/mismatch_intent.meta deleted file mode 100644 index d838db03..00000000 --- a/test/unit_tests/sample_scheme_files/mismatch_intent.meta +++ /dev/null @@ -1,102 +0,0 @@ -[ccpp-table-properties] - name = mismatch_intent - type = scheme - -######################################################################## -[ccpp-arg-table] - name = mismatch_intent_run - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ temp_prev ] - standard_name = potential_temperature_at_previous_timestep - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[ temp_layer ] - standard_name = potential_temperature - units = K - dimensions = () - type = real - kind = kind_phys - intent = in -[ qv ] - standard_name = water_vapor_specific_humidity - units = kg kg-1 - dimensions = (horizontal_loop_extent) - type = real - kind = kind_fizz - intent = inout -[ ps ] - standard_name = surface_air_pressure - state_variable = true - type = integer - kind = kind_phys - units = Pa - dimensions = (horizontal_loop_extent) - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = mismatch_intent_init - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = mismatch_intent_finalize - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/missing_arg_table.F90 b/test/unit_tests/sample_scheme_files/missing_arg_table.F90 deleted file mode 100644 index 9d0a02af..00000000 --- a/test/unit_tests/sample_scheme_files/missing_arg_table.F90 +++ /dev/null @@ -1,75 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE missing_arg_table - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: missing_arg_table_init - PUBLIC :: missing_arg_table_run - PUBLIC :: missing_arg_table_finalize - -CONTAINS - - !> \section arg_table_missing_arg_table_run Argument Table - !! \htmlinclude arg_table_missing_arg_table_run.html - !! - subroutine missing_arg_table_run(foo, timestep, temp_prev, temp_layer, qv, ps, & - errmsg, errflg) - - integer, intent(in) :: foo - real(kind_phys), intent(in) :: timestep - real(kind_phys), intent(inout) :: qv(:) - real(kind_phys), intent(inout) :: ps(:) - REAL(kind_phys), intent(in) :: temp_prev(:) - REAL(kind_phys), intent(inout) :: temp_layer(foo) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: col_index - - errmsg = '' - errflg = 0 - - do col_index = 1, foo - temp_layer(col_index) = temp_layer(col_index) + temp_prev(col_index) - qv(col_index) = qv(col_index) + 1.0_kind_phys - end do - - END SUBROUTINE missing_arg_table_run - - !> \section arg_table_missing_arg_table_init Argument Table - !! \htmlinclude arg_table_missing_arg_table_init.html - !! - subroutine missing_arg_table_init (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine missing_arg_table_init - - !> \section arg_table_missing_arg_table_finalize Argument Table - !! \htmlinclude arg_table_missing_arg_table_finalize.html - !! - subroutine missing_arg_table_finalize (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine missing_arg_table_finalize - -END MODULE missing_arg_table diff --git a/test/unit_tests/sample_scheme_files/missing_fort_header.F90 b/test/unit_tests/sample_scheme_files/missing_fort_header.F90 deleted file mode 100644 index 92981eb5..00000000 --- a/test/unit_tests/sample_scheme_files/missing_fort_header.F90 +++ /dev/null @@ -1,73 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE missing_fort_header - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: missing_fort_header_init - PUBLIC :: missing_fort_header_run - PUBLIC :: missing_fort_header_finalize - -CONTAINS - - !> \section fort_header_missing_arg_table_run Argument Table - !! \htmlinclude fort_header_missing_arg_table_run.html - !! - subroutine missing_fort_header_run(foo, timestep, temp_prev, temp_layer, qv, ps, & - errmsg, errflg) - - integer, intent(in) :: foo - real(kind_phys), intent(in) :: timestep - real(kind_phys), intent(inout) :: qv(:) - real(kind_phys), intent(inout) :: ps(:) - REAL(kind_phys), intent(in) :: temp_prev(:) - REAL(kind_phys), intent(inout) :: temp_layer(foo) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: col_index - - errmsg = '' - errflg = 0 - - do col_index = 1, foo - temp_layer(col_index) = temp_layer(col_index) + temp_prev(col_index) - qv(col_index) = qv(col_index) + 1.0_kind_phys - end do - - END SUBROUTINE missing_fort_header_run - - !> \section fort_header_missing_arg_table_init Argument Table - !! \htmlinclude fort_header_missing_arg_table_init.html - !! - subroutine missing_fort_header_init (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine missing_fort_header_init - - !! - subroutine missing_fort_header_finalize (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine missing_fort_header_finalize - -END MODULE missing_fort_header diff --git a/test/unit_tests/sample_scheme_files/missing_fort_header.meta b/test/unit_tests/sample_scheme_files/missing_fort_header.meta deleted file mode 100644 index d4478ffc..00000000 --- a/test/unit_tests/sample_scheme_files/missing_fort_header.meta +++ /dev/null @@ -1,102 +0,0 @@ -[ccpp-table-properties] - name = missing_fort_header - type = scheme - -######################################################################## -[ccpp-arg-table] - name = missing_fort_header_run - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ temp_prev ] - standard_name = potential_temperature_at_previous_timestep - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[ temp_layer ] - standard_name = potential_temperature - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ qv ] - standard_name = water_vapor_specific_humidity - units = kg kg-1 - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ ps ] - standard_name = surface_air_pressure - state_variable = true - type = real - kind = kind_phys - units = Pa - dimensions = (horizontal_loop_extent) - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = missing_fort_header_init - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = missing_fort_header_finalize - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/reorder.F90 b/test/unit_tests/sample_scheme_files/reorder.F90 deleted file mode 100644 index 690aebe0..00000000 --- a/test/unit_tests/sample_scheme_files/reorder.F90 +++ /dev/null @@ -1,73 +0,0 @@ -MODULE reorder - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: reorder_init - PUBLIC :: reorder_run - PUBLIC :: reorder_finalize - -CONTAINS - - !> \section arg_table_reorder_run Argument Table - !! \htmlinclude arg_table_reorder_run.html - !! - subroutine reorder_run(foo, timestep, temp_prev, temp_layer, qv, ps, & - errmsg, errflg) - - integer, intent(in) :: foo - real(kind_phys), intent(in) :: timestep - real(kind_phys), intent(inout) :: qv(:) - real(kind_phys), intent(inout) :: ps(:) - REAL(kind_phys), intent(in) :: temp_prev(:) - REAL(kind_phys), intent(inout) :: temp_layer(foo) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: col_index - - errmsg = '' - errflg = 0 - - do col_index = 1, foo - temp_layer(col_index) = temp_layer(col_index) + temp_prev(col_index) - qv(col_index) = qv(col_index) + 1.0_kind_phys - end do - - END SUBROUTINE reorder_run - - !> \section arg_table_reorder_init Argument Table - !! \htmlinclude arg_table_reorder_init.html - !! - subroutine reorder_init (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - errmsg = '' - errflg = 0 - - end subroutine reorder_init - - !> \section arg_table_reorder_finalize Argument Table - !! \htmlinclude arg_table_reorder_finalize.html - !! - subroutine reorder_finalize (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - errmsg = '' - errflg = 0 - - end subroutine reorder_finalize - -END MODULE reorder - - ! add some stuff here to check if codee really ignores this - - -! BLA \ No newline at end of file diff --git a/test/unit_tests/sample_scheme_files/reorder.meta b/test/unit_tests/sample_scheme_files/reorder.meta deleted file mode 100644 index e69f6f8d..00000000 --- a/test/unit_tests/sample_scheme_files/reorder.meta +++ /dev/null @@ -1,102 +0,0 @@ -[ccpp-table-properties] - name = reorder - type = scheme - -######################################################################## -[ccpp-arg-table] - name = reorder_finalize - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = reorder_run - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ temp_prev ] - standard_name = potential_temperature_at_previous_timestep - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[ temp_layer ] - standard_name = potential_temperature - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ qv ] - standard_name = water_vapor_specific_humidity - units = kg kg-1 - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ ps ] - standard_name = surface_air_pressure - state_variable = true - type = real - kind = kind_phys - units = Pa - dimensions = (horizontal_loop_extent) - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = reorder_init - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/temp_adjust.F90 b/test/unit_tests/sample_scheme_files/temp_adjust.F90 deleted file mode 100644 index 70613ba1..00000000 --- a/test/unit_tests/sample_scheme_files/temp_adjust.F90 +++ /dev/null @@ -1,96 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE temp_adjust - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: temp_adjust_init - PUBLIC :: temp_adjust_run - PUBLIC :: temp_adjust_finalize - -CONTAINS - - !> \section arg_table_temp_adjust_register Argument Table - !! \htmlinclude arg_table_temp_adjust_register.html - !! - subroutine temp_adjust_register(config_var, dyn_const, errflg, errmsg) - logical, intent(in) :: config_var - type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - if (.not. config_var) then - return - end if - - allocate(dyn_const(1)) - call dyn_const(1)%instantiate(std_name="dyn_const", long_name='dyn const', & - diag_name='DYNCONST', units='kg kg-1', default_value=1._kind_phys, & - vertical_dim='vertical_layer_dimension', advected=.true., & - errcode=errflg, errmsg=errmsg) - - end subroutine temp_adjust_register - - !> \section arg_table_temp_adjust_run Argument Table - !! \htmlinclude arg_table_temp_adjust_run.html - !! - subroutine temp_adjust_run(foo, timestep, temp_prev, temp_layer, qv, ps, & - errmsg, errflg) - - integer, intent(in) :: foo - real(kind_phys), intent(in) :: timestep - real(kind_phys), intent(inout) :: qv(:) - real(kind_phys), intent(inout) :: ps(:) - REAL(kind_phys), intent(in) :: temp_prev(:) - REAL(kind_phys), intent(inout) :: temp_layer(foo) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: col_index - - errmsg = '' - errflg = 0 - - do col_index = 1, foo - temp_layer(col_index) = temp_layer(col_index) + temp_prev(col_index) - qv(col_index) = qv(col_index) + 1.0_kind_phys - end do - - END SUBROUTINE temp_adjust_run - - !> \section arg_table_temp_adjust_init Argument Table - !! \htmlinclude arg_table_temp_adjust_init.html - !! - subroutine temp_adjust_init (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine temp_adjust_init - - !> \section arg_table_temp_adjust_finalize Argument Table - !! \htmlinclude arg_table_temp_adjust_finalize.html - !! - subroutine temp_adjust_finalize (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine temp_adjust_finalize - -END MODULE temp_adjust diff --git a/test/unit_tests/test_common.py b/test/unit_tests/test_common.py deleted file mode 100644 index aa81ee8d..00000000 --- a/test/unit_tests/test_common.py +++ /dev/null @@ -1,92 +0,0 @@ -#! /usr/bin/env python3 -""" ------------------------------------------------------------------------ - Description: Contains unit tests for functions in common.py - - Assumptions: - - Command line arguments: none - - Usage: python test_common.py # run the unit tests ------------------------------------------------------------------------ -""" -import sys -import os -import logging -import unittest - -TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_FILE = os.path.abspath(os.path.abspath(__file__)) -SCRIPTS_DIR = os.path.abspath(os.path.join(TEST_DIR, os.pardir, os.pardir, "scripts")) -SAMPLE_FILES_DIR = os.path.join(TEST_DIR, "sample_files") - -if not os.path.exists(SCRIPTS_DIR): - raise ImportError("Cannot find scripts directory") - -sys.path.append(SCRIPTS_DIR) -logging.disable(level=logging.CRITICAL) - -# pylint: disable=wrong-import-position -import common -# pylint: enable=wrong-import-position - -class CommonTestCase(unittest.TestCase): - - """Tests functionality of functions in common.py""" - - def test_split_var_name_and_array_reference(self): - """Test split_var_name_and_array_reference() function""" - - self.assertEqual(common.split_var_name_and_array_reference("foo(:,a,1:ddt%ngas)"), - ("foo","(:,a,1:ddt%ngas)")) - - def test_encode_container(self): - """Test encode_container() function""" - - modulename = "ABCD1234" - typename = "COMPLEX" - schemename = "testscheme" - subroutinename = "testsubroutine" - self.assertEqual(common.encode_container(modulename),f"MODULE_{modulename.lower()}") - self.assertEqual(common.encode_container(modulename,typename),f"MODULE_{modulename.lower()} TYPE_{typename.lower()}") - self.assertEqual(common.encode_container(modulename,schemename,subroutinename), - f"MODULE_{modulename.lower()} SCHEME_{schemename.lower()} SUBROUTINE_{subroutinename.lower()}") - self.assertRaises(Exception,common.encode_container,modulename,typename,schemename,subroutinename) - self.assertRaises(Exception,common.encode_container) - - def test_decode_container(self): - """Test decode_container() function""" - - modulename = "abcd1234" - typename = "complex" - schemename = "testscheme" - subroutinename = "testsubroutine" - self.assertEqual(common.decode_container(f"MODULE_{modulename}"),f"MODULE {modulename}") - self.assertEqual(common.decode_container(f"MODULE_{modulename} TYPE_{typename}"), - f"MODULE {modulename} TYPE {typename}") - self.assertEqual(common.decode_container(f"MODULE_{modulename} SCHEME_{schemename} SUBROUTINE_{subroutinename}"), - f"MODULE {modulename} SCHEME {schemename} SUBROUTINE {subroutinename}") - self.assertRaises(Exception,common.decode_container, - f"MODULE_{modulename} TYPE_{typename} SCHEME_{schemename} SUBROUTINE_{subroutinename}") - self.assertRaises(Exception,common.decode_container,"That dog won't hunt, Monsignor") - self.assertRaises(Exception,common.decode_container) - - def test_string_to_python_identifier(self): - """Test string_to_python_identifier() function""" - - # Test various successful combinations - self.assertEqual(common.string_to_python_identifier("Test 1"),"Test_1") - self.assertEqual(common.string_to_python_identifier("Test.2"),"Test_p_2") - self.assertEqual(common.string_to_python_identifier("Test-3"),"Test_minus_3") - self.assertEqual(common.string_to_python_identifier("Test+4"),"Test_plus_4") - self.assertEqual(common.string_to_python_identifier("1"),"one") - self.assertEqual(common.string_to_python_identifier(" Test all--even +."), - "_Test_all_minus__minus_even__plus__p_") - # Test expected failures - self.assertRaises(Exception,common.string_to_python_identifier,"else") - self.assertRaises(Exception,common.string_to_python_identifier,"1 ") - self.assertRaises(Exception,common.string_to_python_identifier,"0") - self.assertRaises(Exception,common.string_to_python_identifier,"Disallowed character!") - -if __name__ == '__main__': - unittest.main() diff --git a/test/unit_tests/test_fortran_parse.py b/test/unit_tests/test_fortran_parse.py deleted file mode 100644 index 52696cd5..00000000 --- a/test/unit_tests/test_fortran_parse.py +++ /dev/null @@ -1,101 +0,0 @@ -#! /usr/bin/env python3 -""" ------------------------------------------------------------------------ - Description: Contains unit tests for parsing Fortran - in scripts files scripts/fortran_tools/parse_fortran_file.py and - scripts/fortran_tools/parse_fortran.py - - Assumptions: - - Command line arguments: none - - Usage: python3 test_fortran_parse.py # run the unit tests ------------------------------------------------------------------------ -""" - -import os -import sys -import logging -import unittest - -_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -_SCRIPTS_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, - os.pardir, "scripts")) -_SAMPLE_FILES_DIR = os.path.join(_TEST_DIR, "sample_files", "fortran_files") -_PRE_TMP_DIR = os.path.join(_TEST_DIR, "tmp") -_TMP_DIR = os.path.join(_PRE_TMP_DIR, "fortran_files") - -if not os.path.exists(_SCRIPTS_DIR): - raise ImportError(f"Cannot find scripts directory, {_SCRIPTS_DIR}") - -sys.path.append(_SCRIPTS_DIR) - -# pylint: disable=wrong-import-position -from fortran_tools import parse_fortran_file -from framework_env import CCPPFrameworkEnv -# pylint: enable=wrong-import-position - -############################################################################### -def remove_files(file_list): -############################################################################### - """Remove files in if they exist""" - if isinstance(file_list, str): - file_list = [file_list] - # end if - for fpath in file_list: - if os.path.exists(fpath): - os.remove(fpath) - # End if - # End for - -class FortranParseTestCase(unittest.TestCase): - - """Tests for `parse_fortran_file`.""" - - _run_env = None - - @classmethod - def setUpClass(cls): - """Clean output directory (tmp) before running tests""" - # Does "tmp" directory exist? If not then create it: - if not os.path.exists(_PRE_TMP_DIR): - os.makedirs(_PRE_TMP_DIR) - # end if - - # We need a run environment - logger = logging.getLogger(cls.__name__) - cls._run_env = CCPPFrameworkEnv(logger, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - - # Run inherited setup method: - super().setUpClass() - - def test_array_parsing(self): - """Test that the Fortran parser outputs an informative - error message for a badly formatted array specification. - Also, test that allowed specification strings are allowed. - """ - # Setup - testname = "array_parsing_test" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.F90") - # Exercise - header = "Test of parsing of Fortran array specification" - - with self.assertRaises(Exception) as context: - # Parse the file - _ = parse_fortran_file(source, self._run_env) - # end if - - # Check exception for expected error messages - self.assertTrue("bad_arr1: ';' is not a valid Fortran identifier" - in str(context.exception)) - self.assertTrue("bad_arr2: ';' is not a valid Fortran identifier" - in str(context.exception)) - self.assertTrue("bad_arr3: ';' is not a valid Fortran identifier" - in str(context.exception)) - self.assertTrue("Missing local_variables, ['bad_arr1', 'bad_arr2', 'bad_arr3'] in array_spec_test_run" - in str(context.exception)) - -if __name__ == "__main__": - unittest.main() diff --git a/test/unit_tests/test_fortran_write.py b/test/unit_tests/test_fortran_write.py deleted file mode 100644 index 49f551f7..00000000 --- a/test/unit_tests/test_fortran_write.py +++ /dev/null @@ -1,172 +0,0 @@ -#! /usr/bin/env python3 -""" ------------------------------------------------------------------------ - Description: Contains unit tests for FortranWriter - in scripts file fortran/fortran_write.py - - Assumptions: - - Command line arguments: none - - Usage: python3 test_fortran_write.py # run the unit tests ------------------------------------------------------------------------ -""" - -import filecmp -import glob -import os -import sys -import unittest - -_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -_SCRIPTS_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, - os.pardir, "scripts")) -_SAMPLE_FILES_DIR = os.path.join(_TEST_DIR, "sample_files", "fortran_files") -_PRE_TMP_DIR = os.path.join(_TEST_DIR, "tmp") -_TMP_DIR = os.path.join(_PRE_TMP_DIR, "fortran_files") - -if not os.path.exists(_SCRIPTS_DIR): - raise ImportError(f"Cannot find scripts directory, {_SCRIPTS_DIR}") - -sys.path.append(_SCRIPTS_DIR) - -# pylint: disable=wrong-import-position -from fortran_tools import FortranWriter -# pylint: enable=wrong-import-position - -############################################################################### -def remove_files(file_list): -############################################################################### - """Remove files in if they exist""" - if isinstance(file_list, str): - file_list = [file_list] - # end if - for fpath in file_list: - if os.path.exists(fpath): - os.remove(fpath) - # End if - # End for - -class FortranWriterTestCase(unittest.TestCase): - - """Tests for `FortranWriter`.""" - - @classmethod - def setUpClass(cls): - """Clean output directory (tmp) before running tests""" - #Does "tmp" directory exist? If not then create it: - if not os.path.exists(_PRE_TMP_DIR): - os.mkdir(_PRE_TMP_DIR) - # Ensure the "sample_files/fortran_files" directory exists and is empty - if os.path.exists(_TMP_DIR): - # Clear out all files: - remove_files(glob.iglob(os.path.join(_TMP_DIR, '*.*'))) - else: - os.makedirs(_TMP_DIR) - # end if - - #Run inherited setup method: - super().setUpClass() - - def test_line_breaking(self): - """Test that FortranWriter correctly breaks long lines""" - # Setup - testname = "linebreak_test" - compare = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.F90") - generate = os.path.join(_TMP_DIR, f"{testname}.F90") - # Exercise - header = "Test of line breaking for FortranWriter" - with FortranWriter(generate, 'w', header, f"{testname}") as gen: - # Test long declaration - qchr = "'" - nditems = 100 - data_items = ', '.join([f"{qchr}name{x:03}{qchr}" for x in range(nditems)]) - gen.write(f"character(len=7) :: data({nditems}) = (/ {data_items} /)", 1) - gen.end_module_header() - # Test long code lines - gen.blank_line() - gen.write("subroutine foo(ozone_constituents, aerosol_constituents, " - "volcaero_constituents, other_constituents)", 1) - gen.write("integer, intent(in) :: ozone_constituents(:)", 2) - gen.write("integer, intent(in) :: aerosol_constituents(:)", 2) - gen.write("integer, intent(in) :: volcaero_constituents(:)", 2) - gen.write("integer, intent(in) :: other_constituents(:)", 2) - gen.write("real, allocatable :: tracer_data_test_dynamic_constituents(:)", 2) - gen.comment("codee format off", 0) - gen.write("allocate(tracer_data_test_dynamic_constituents(0+" - "size(ozone_constituents)+size(aerosol_constituents)" - "+size(volcaero_constituents)+size(other_constituents)))", 2) - gen.blank_line() - line_items = ["write(6, '(a)') 'Cannot read columns_on_task from ", - "file'//', columns_on_task has no horizontal ", - "dimension; columns_on_task is a ", - "protected variable'"] - gen.write(f"{''.join(line_items)}", 2) - gen.comment("codee format on", 0) - gen.write("end subroutine foo", 1) - # end with - - # Check that file was generated - amsg = f"{generate} does not exist" - self.assertTrue(os.path.exists(generate), msg=amsg) - amsg = f"{generate} does not match {compare}" - self.assertTrue(filecmp.cmp(generate, compare, shallow=False), msg=amsg) - - def test_good_comments(self): - """Test that comments are written and broken correctly.""" - # Setup - testname = "comments_test" - compare = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.F90") - generate = os.path.join(_TMP_DIR, f"{testname}.F90") - # Exercise - header = "Test of comment writing for FortranWriter" - with FortranWriter(generate, 'w', header, f"{testname}") as gen: - gen.comment("codee format off", 0) - gen.comment("We can write comments in the module header", 0) - gen.comment("codee format on", 0) - gen.comment("We can write indented comments in the header", 1) - gen.write("integer :: foo ! Comment at end of line works", 1) - # Test long comments at end of line - gen.write(f"integer :: bar ! {'x'*100}", 1) - gen.write(f"integer :: baz ! {'y'*130}", 1) - gen.end_module_header() - # Test comment line in body - gen.comment("We can write comments in the module body", 1) - - # end with - - # Check that file was generated - amsg = f"{generate} does not exist" - self.assertTrue(os.path.exists(generate), msg=amsg) - amsg = f"{generate} does not match {compare}" - self.assertTrue(filecmp.cmp(generate, compare, shallow=False), msg=amsg) - - def test_long_strings(self): - """Test breaking of long strings""" - # Setup - testname = "long_string_test" - compare = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.F90") - generate = os.path.join(_TMP_DIR, f"{testname}.F90") - # Exercise - header = "Test of long string breaking for FortranWriter" - foostr = '0123456789'*10 - nxtchr = ord('0') - with FortranWriter(generate, 'w', header, f"{testname}") as gen: - while len(foostr) < 130: - gen.write(f"character(len={len(foostr)}) :: foo{len(foostr)} = '{foostr.strip()}'", 1) - foostr += chr(nxtchr) - nxtchr += 1 - if nxtchr > ord('9'): - nxtchr = ord('0') - # end if - # end while - # end with - - # Check that file was generated - amsg = f"{generate} does not exist" - self.assertTrue(os.path.exists(generate), msg=amsg) - amsg = f"{generate} does not match {compare}" - self.assertTrue(filecmp.cmp(generate, compare, shallow=False), msg=amsg) - -if __name__ == "__main__": - unittest.main() diff --git a/test/unit_tests/test_metadata_host_file.py b/test/unit_tests/test_metadata_host_file.py deleted file mode 100644 index 53e0a9fe..00000000 --- a/test/unit_tests/test_metadata_host_file.py +++ /dev/null @@ -1,280 +0,0 @@ -#! /usr/bin/env python3 - -""" ------------------------------------------------------------------------ - Description: capgen needs to compare a metadata header against the - associated CCPP Fortran interface routine. This set of - tests is testing the parse_host_model_files function in - ccpp_capgen.py which performs the operations in the first - bullet below. Each test calls this function. - - * This script contains unit tests that do the following: - 1) Read one or more metadata files (to collect - the metadata headers) - 2) Read the associated CCPP Fortran host file(s) (to - collect Fortran interfaces) - 3) Create a CCPP host model object - 3) Test the properties of the CCPP host model object - - * Tests include: - - Correctly parse and match a simple module file with - data fields (a data block) - - Correctly parse and match a simple module file with - a DDT definition - - Correctly parse and match a simple module file with - two DDT definitions - - Correctly parse and match a simple module file with - two DDT definitions and a data block - - "Test correct use of loop variables (horizontal - dimensions) in host metadata - - Assumptions: - - Command line arguments: none - - Usage: python3 test_metadata_host_file.py # run the unit tests ------------------------------------------------------------------------ -""" -import sys -import os -import logging -import unittest - -_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -_SCRIPTS_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, - os.pardir, "scripts")) -if not os.path.exists(_SCRIPTS_DIR): - raise ImportError("Cannot find scripts directory") - -sys.path.append(_SCRIPTS_DIR) - -# pylint: disable=wrong-import-position -from ccpp_capgen import parse_host_model_files -from framework_env import CCPPFrameworkEnv -from parse_tools import CCPPError -# pylint: enable=wrong-import-position - -class MetadataHeaderTestCase(unittest.TestCase): - """Unit tests for parse_host_model_files""" - - def setUp(self): - """Setup important directories and logging""" - self._sample_files_dir = os.path.join(_TEST_DIR, "sample_host_files") - logger = logging.getLogger(self.__class__.__name__) - self._run_env = CCPPFrameworkEnv(logger, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - - def test_module_with_data(self): - """Test that a module containing a data block is parsed and matched - correctly.""" - # Setup - module_files = [os.path.join(self._sample_files_dir, "data1_mod.meta")] - # Exercise - hname = 'host_name_data1' - host_model = parse_host_model_files(module_files, hname, self._run_env) - # Verify the name of the host model - self.assertEqual(host_model.name, hname) - module_headers = host_model.metadata_tables() - self.assertEqual(len(module_headers), 1) - # Verify header titles - self.assertTrue('data1_mod' in module_headers) - # Verify host model variable list - vlist = host_model.variable_list() - self.assertEqual(len(vlist), 3) - std_names = [x.get_prop_value('standard_name') for x in vlist] - self.assertTrue('play_station' in std_names) - self.assertTrue('xbox' in std_names) - self.assertTrue('nintendo_switch' in std_names) - - def test_module_with_one_ddt(self): - """Test that a module containing a DDT definition is parsed and matched - correctly.""" - # Setup - ddt_name = 'ddt1_t' - module_files = [os.path.join(self._sample_files_dir, "ddt1.meta")] - # Exercise - hname = 'host_name_ddt1' - host_model = parse_host_model_files(module_files, hname, self._run_env) - # Verify the name of the host model - self.assertEqual(host_model.name, hname) - module_headers = host_model.metadata_tables() - self.assertEqual(len(module_headers), 1) - # Verify header titles - self.assertTrue(ddt_name in module_headers) - # Verify host model variable list - vlist = host_model.variable_list() - self.assertEqual(len(vlist), 0) - # Verify that the DDT was found and parsed - ddt_lib = host_model.ddt_lib - self.assertEqual(ddt_lib.name, f"{hname}_ddts_ddt_lib") - # Check DDT variables - ddt_mod = ddt_lib[ddt_name] - self.assertEqual(ddt_mod.name, ddt_name) - vlist = ddt_mod.variable_list() - self.assertEqual(len(vlist), 2) - std_names = [x.get_prop_value('standard_name') for x in vlist] - self.assertTrue('ddt_var_array_dimension' in std_names) - self.assertTrue('vars_array' in std_names) - - def test_module_with_two_ddts(self): - """Test that a module containing two DDT definitions is parsed and - matched correctly.""" - # Setup - ddt_names = ['ddt1_t', 'ddt2_t'] - ddt_vars = [(), ('ddt_var_array_dimension', 'vars_array')] - - module_files = [os.path.join(self._sample_files_dir, "ddt2.meta")] - # Exercise - hname = 'host_name_ddt2' - host_model = parse_host_model_files(module_files, hname, self._run_env) - # Verify the name of the host model - self.assertEqual(host_model.name, hname) - module_headers = host_model.metadata_tables() - self.assertEqual(len(module_headers), len(ddt_names)) - # Verify header titles - for ddt_name in ddt_names: - self.assertTrue(ddt_name in module_headers) - # end for - # Verify host model variable list - vlist = host_model.variable_list() - self.assertEqual(len(vlist), 0) - # Verify that each DDT was found and parsed - ddt_lib = host_model.ddt_lib - self.assertEqual(ddt_lib.name, f"{hname}_ddts_ddt_lib") - # Check DDT variables - for index, ddt_name in enumerate(ddt_names): - ddt_mod = ddt_lib[ddt_name] - self.assertEqual(ddt_mod.name, ddt_name) - vlist = ddt_mod.variable_list() - self.assertEqual(len(vlist), len(ddt_vars[index])) - std_names = [x.get_prop_value('standard_name') for x in vlist] - for sname in ddt_vars[index]: - self.assertTrue(sname in std_names) - # end for - # end for - - def test_module_with_two_ddts_and_data(self): - """Test that a module containing two DDT definitions and a block of - module data is parsed and matched correctly.""" - # Setup - ddt_names = ['ddt1_t', 'ddt2_t'] - ddt_vars = [(), ('ddt_var_array_dimension', 'vars_array')] - - module_files = [os.path.join(self._sample_files_dir, - "ddt_data1_mod.meta")] - # Exercise - hname = 'host_name_ddt_data' - host_model = parse_host_model_files(module_files, hname, self._run_env) - # Verify the name of the host model - self.assertEqual(host_model.name, hname) - module_headers = host_model.metadata_tables() - self.assertEqual(len(module_headers), len(ddt_names) + 1) - # Verify header titles - for ddt_name in ddt_names: - self.assertTrue(ddt_name in module_headers) - # end for - # Verify host model variable list - vlist = host_model.variable_list() - self.assertEqual(len(vlist), 3) - # Verify that each DDT was found and parsed - ddt_lib = host_model.ddt_lib - self.assertEqual(ddt_lib.name, f"{hname}_ddts_ddt_lib") - # Check DDT variables - for index, ddt_name in enumerate(ddt_names): - ddt_mod = ddt_lib[ddt_name] - self.assertEqual(ddt_mod.name, ddt_name) - vlist = ddt_mod.variable_list() - self.assertEqual(len(vlist), len(ddt_vars[index])) - std_names = [x.get_prop_value('standard_name') for x in vlist] - for sname in ddt_vars[index]: - self.assertTrue(sname in std_names) - # end for - # end for - # Verify header titles - self.assertTrue('ddt_data1_mod' in module_headers) - # Verify host model variable list - vlist = host_model.variable_list() - self.assertEqual(len(vlist), 3) - std_names = [x.get_prop_value('standard_name') for x in vlist] - self.assertTrue('play_station' in std_names) - self.assertTrue('xbox' in std_names) - self.assertTrue('nintendo_switch' in std_names) - - def test_module_with_one_ddt_plus_undoc(self): - """Test that a module containing a one documented DDT definition - (i.e., with metadata) and one DDT without (i.e., no metadata) - is parsed and matched correctly.""" - # Setup - ddt_name = 'ddt2_t' - module_files = [os.path.join(self._sample_files_dir, "ddt1_plus.meta")] - # Exercise - hname = 'host_name_ddt1_plus' - host_model = parse_host_model_files(module_files, hname, self._run_env) - # Verify the name of the host model - self.assertEqual(host_model.name, hname) - module_headers = host_model.metadata_tables() - self.assertEqual(len(module_headers), 1) - # Verify header titles - self.assertTrue(ddt_name in module_headers) - # Verify host model variable list - vlist = host_model.variable_list() - self.assertEqual(len(vlist), 0) - # Verify that the DDT was found and parsed - ddt_lib = host_model.ddt_lib - self.assertEqual(ddt_lib.name, f"{hname}_ddts_ddt_lib") - # Check DDT variables - ddt_mod = ddt_lib[ddt_name] - self.assertEqual(ddt_mod.name, ddt_name) - vlist = ddt_mod.variable_list() - self.assertEqual(len(vlist), 2) - std_names = [x.get_prop_value('standard_name') for x in vlist] - self.assertTrue('ddt_var_array_dimension' in std_names) - self.assertTrue('vars_array' in std_names) - - def test_module_with_two_ddts_and_extra_var(self): - """Test that a module containing two DDT definitions is parsed and - a useful error message is produced if the DDT metadata has an - extra variable.""" - # Setup - ddt_names = ['ddt1_t', 'ddt2_t'] - ddt_vars = [(), ('ddt_var_array_dimension', 'vars_array')] - - module_files = [os.path.join(self._sample_files_dir, - "ddt2_extra_var.meta")] - # Exercise - hname = 'host_name_ddt_extra_var' - with self.assertRaises(CCPPError) as context: - host_model = parse_host_model_files(module_files, hname, - self._run_env) - # end with - # Check error messages - except_str = str(context.exception) - emsgs = ["Variable mismatch in ddt2_t, variables missing from Fortran ddt.", - "No Fortran variable for bogus in ddt2_t", - "2 errors found comparing"] - for emsg in emsgs: - self.assertTrue(emsg in except_str) - # end for - - def test_mismatch_hdim(self): - """Test correct use of loop variables (horizontal dimensions) - in host metadata.""" - # Setup - module_files = [os.path.join(self._sample_files_dir, "mismatch_hdim_mod.meta")] - # Exercise - hname = 'host_name_mismatch_hdim' - with self.assertRaises(CCPPError) as context: - _ = parse_host_model_files(module_files, hname, self._run_env) - # end with - # Check error messages - except_str = str(context.exception) - emsgs = ["Invalid horizontal dimension, 'horizontal_loop_extent'", - "Invalid horizontal dimension, 'horizontal_loop_end'"] - for emsg in emsgs: - self.assertTrue(emsg in except_str) - # end for - -if __name__ == "__main__": - unittest.main() - diff --git a/test/unit_tests/test_metadata_scheme_file.py b/test/unit_tests/test_metadata_scheme_file.py deleted file mode 100644 index ef24742c..00000000 --- a/test/unit_tests/test_metadata_scheme_file.py +++ /dev/null @@ -1,357 +0,0 @@ -#! /usr/bin/env python3 - -""" ------------------------------------------------------------------------ - Description: capgen needs to compare a metadata header against the - associated CCPP Fortran interface routine. This set of - tests is testing the parse_scheme_files function in - ccpp_capgen.py which performs the operations in the first - bullet below. Each test calls this function. - - * This script contains unit tests that do the following: - 1) Read a metadata file (to collect the metadata headers) - 2) Read the associated CCPP Fortran scheme file (to - collect Fortran interfaces) - 3) Compare the metadata header against the Fortran - - * Tests include: - - Correctly identify when the metadata file matches the - Fortran, even if the routines are not in the same order - - Correctly detect a missing metadata header - - Correctly detect a missing Fortran interface - - Correctly detect a mismatch between the metadata and the - Fortran - - Correctly detect invalid Fortran subroutine statements, - invalid dummy argument statements, and invalid Fortran - between the subroutine statement and the end of the - variable declaration block. - - Correctly interpret Fortran with preprocessor logic - which affects the subroutine statement and/or the dummy - argument statements - - Correctly interpret Fortran with preprocessor logic - which affects the subroutine statement and/or the dummy - argument statements resulting in a mismatch between the - metadata header and the Fortran - - Correctly interpret Fortran with preprocessor logic - which affects the subroutine statement and/or the dummy - argument statements resulting in incorrect Fortran - - Test correct use of loop variables (horizontal dimensions) - in scheme metadata. The allowed values depend on the phase - (run phase or not) - - Assumptions: - - Command line arguments: none - - Usage: python3 test_metadata_scheme_file.py # run the unit tests ------------------------------------------------------------------------ -""" -import sys -import os -import logging -import unittest - -_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -_SCRIPTS_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, - os.pardir, "scripts")) -if not os.path.exists(_SCRIPTS_DIR): - raise ImportError("Cannot find scripts directory") - -sys.path.append(_SCRIPTS_DIR) - -# pylint: disable=wrong-import-position -from ccpp_capgen import parse_scheme_files -from framework_env import CCPPFrameworkEnv -from parse_tools import CCPPError -# pylint: enable=wrong-import-position - -class MetadataHeaderTestCase(unittest.TestCase): - """Unit tests for parse_scheme_files""" - - def setUp(self): - """Setup important directories and logging""" - self._sample_files_dir = os.path.join(_TEST_DIR, "sample_scheme_files") - self._host_files_dir = os.path.join(_TEST_DIR, "sample_host_files") - logger = logging.getLogger(self.__class__.__name__) - self._run_env = CCPPFrameworkEnv(logger, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - self._run_env_ccpp = CCPPFrameworkEnv(logger, - ndict={'host_files':'', - 'scheme_files':'', - 'suites':'', - 'preproc_directives': - 'CCPP=1'}) - self._run_env_ccpp2 = CCPPFrameworkEnv(logger, - ndict={'host_files':'', - 'scheme_files':'', - 'suites':'', - 'preproc_directives': - 'CCPP=2'}) - - def test_good_scheme_file(self): - """Test that good metadata file matches the Fortran, - with routines in the same order """ - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "temp_adjust.meta")] - # Exercise - scheme_headers, table_dict = parse_scheme_files(scheme_files, - self._run_env, - skip_ddt_check=True) - # Verify size of returned list equals number of scheme headers - # in the test file and that header (subroutine) names are - # 'temp_adjust_[register,init,run,finalize]' - self.assertEqual(len(scheme_headers), 4) - # Verify header titles - titles = [elem.title for elem in scheme_headers] - self.assertTrue('temp_adjust_register' in titles) - self.assertTrue('temp_adjust_init' in titles) - self.assertTrue('temp_adjust_run' in titles) - self.assertTrue('temp_adjust_finalize' in titles) - # Verify size and name of table_dict matches scheme name - self.assertEqual(len(table_dict), 1) - self.assertTrue('temp_adjust' in table_dict) - - def test_reordered_scheme_file(self): - """Test that metadata file matches the Fortran when the - routines are not in the same order """ - # Setup - scheme_files = [os.path.join(self._sample_files_dir, "reorder.meta")] - # Exercise - scheme_headers, table_dict = parse_scheme_files(scheme_files, - self._run_env) - # Verify size of returned list equals number of scheme headers - # in the test file and that header (subroutine) names are - # 'reorder_[init,run,finalize]' - self.assertEqual(len(scheme_headers), 3) - # Verify header titles - titles = [elem.title for elem in scheme_headers] - self.assertTrue('reorder_init' in titles) - self.assertTrue('reorder_run' in titles) - self.assertTrue('reorder_finalize' in titles) - # Verify size and name of table_dict matches scheme name - self.assertEqual(len(table_dict), 1) - self.assertTrue('reorder' in table_dict) - - def test_missing_metadata_header(self): - """Test that a missing metadata header (aka arg table) is - corretly detected """ - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "missing_arg_table.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - parse_scheme_files(scheme_files, self._run_env) - # Verify correct error message returned - emsg = "No matching metadata header found for missing_arg_table_run in" - self.assertTrue(emsg in str(context.exception)) - - def test_missing_fortran_header(self): - """Test that a missing fortran header is corretly detected """ - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "missing_fort_header.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - parse_scheme_files(scheme_files, self._run_env) - # Verify correct error message returned - emsg = "No matching Fortran routine found for missing_fort_header_run in" - self.assertTrue(emsg in str(context.exception)) - - def test_mismatch_intent(self): - """Test that differing intent, kind, rank, and type between - metadata and fortran is corretly detected """ - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "mismatch_intent.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - parse_scheme_files(scheme_files, self._run_env) - # Verify 4 correct error messages returned - emsg = "intent mismatch (in != inout) in mismatch_intent_run, at" - self.assertTrue(emsg in str(context.exception)) - emsg = "kind mismatch (kind_fizz != kind_phys) in mismatch_intent_run, at" - self.assertTrue(emsg in str(context.exception)) - emsg = "rank mismatch in mismatch_intent_run/potential_temperature (0 != 1), at" - self.assertTrue(emsg in str(context.exception)) - emsg = "type mismatch (integer != real) in mismatch_intent_run, at" - self.assertTrue(emsg in str(context.exception)) - self.assertTrue("4 errors found comparing" in str(context.exception)) - - def test_invalid_subr_stmnt(self): - """Test that invalid Fortran subroutine statements are correctly - detected """ - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "invalid_subr_stmnt.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - parse_scheme_files(scheme_files, self._run_env) - # Verify correct error message returned - self.assertTrue("Invalid dummy argument, 'errmsg', at" - in str(context.exception)) - - def test_invalid_dummy_arg(self): - """Test that invalid dummy argument statements are correctly detected""" - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "invalid_dummy_arg.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - parse_scheme_files(scheme_files, self._run_env) - # Verify correct error message returned - emsg = "Invalid dummy argument, 'woohoo', at" - self.assertTrue(emsg in str(context.exception)) - - def test_ccpp_notset_var_missing_in_meta(self): - """Test for correct detection of a variable that REMAINS in the - subroutine argument list - (due to an undefined pre-processor directive: #ifndef CCPP), - BUT IS NOT PRESENT in meta file""" - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "CCPPnotset_var_missing_in_meta.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - parse_scheme_files(scheme_files, self._run_env) - # Verify 2 correct error messages returned - emsg = "Variable mismatch in CCPPnotset_var_missing_in_meta_run, " + \ - "variables missing from metadata header." - self.assertTrue(emsg in str(context.exception)) - emsg = "Fortran variable, bar, not in metadata" - self.assertTrue(emsg in str(context.exception)) - self.assertTrue("2 errors found comparing" in str(context.exception)) - - def test_ccpp_eq1_var_missing_in_fort(self): - """Test for correct detection of a variable that IS REMOVED the - subroutine argument list - (due to a pre-processor directive: #ifndef CCPP), - but IS PRESENT in meta file""" - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "CCPPeq1_var_missing_in_fort.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - parse_scheme_files(scheme_files, self._run_env_ccpp) - # Verify 2 correct error messages returned - emsg = "Variable mismatch in CCPPeq1_var_missing_in_fort_run, " + \ - "variables missing from Fortran scheme." - self.assertTrue(emsg in str(context.exception)) - emsg = "Variable mismatch in CCPPeq1_var_missing_in_fort_run, " + \ - "no Fortran variable bar." - self.assertTrue(emsg in str(context.exception)) - self.assertTrue("2 errors found comparing" in str(context.exception)) - - def test_ccpp_eq1_var_in_fort_meta(self): - """Test positive case of a variable that IS PRESENT the - subroutine argument list - (due to a pre-processor directive: #ifdef CCPP), - and IS PRESENT in meta file""" - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "CCPPeq1_var_in_fort_meta.meta")] - # Exercise - scheme_headers, table_dict = parse_scheme_files(scheme_files, - self._run_env_ccpp) - # Verify size of returned list equals number of scheme headers in - # the test file (1) and that header (subroutine) name is - # 'CCPPeq1_var_in_fort_meta_run' - self.assertEqual(len(scheme_headers), 1) - # Verify header titles - titles = [elem.title for elem in scheme_headers] - self.assertTrue("CCPPeq1_var_in_fort_meta_run" in titles) - - # Verify size and name of table_dict matches scheme name - self.assertEqual(len(table_dict), 1) - self.assertTrue("CCPPeq1_var_in_fort_meta" in table_dict) - - def test_ccpp_gt1_var_in_fort_meta(self): - """Test positive case of a variable that IS PRESENT the - subroutine argument list - (due to a pre-processor directive: #if CCPP > 1), - and IS PRESENT in meta file""" - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "CCPPgt1_var_in_fort_meta.meta")] - # Exercise - # Set CCPP directive to > 1 - scheme_headers, table_dict = parse_scheme_files(scheme_files, - self._run_env_ccpp2) - # Verify size of returned list equals number of scheme headers - # in the test file (1) and that header (subroutine) name is - # 'CCPPgt1_var_in_fort_meta_init' - self.assertEqual(len(scheme_headers), 1) - # Verify header titles - titles = [elem.title for elem in scheme_headers] - self.assertTrue("CCPPgt1_var_in_fort_meta_init" in titles) - - # Verify size and name of table_dict matches scheme name - self.assertEqual(len(table_dict), 1) - self.assertTrue("CCPPgt1_var_in_fort_meta" in table_dict) - - def test_ccpp_gt1_var_in_fort_meta2(self): - """Test correct detection of a variable that - IS NOT PRESENT the subroutine argument list - (due to a pre-processor directive: #if CCPP > 1), - but IS PRESENT in meta file""" - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "CCPPgt1_var_in_fort_meta.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - _, _ = parse_scheme_files(scheme_files, self._run_env_ccpp) - # Verify 2 correct error messages returned - emsg = "Variable mismatch in CCPPgt1_var_in_fort_meta_init, " + \ - "variables missing from Fortran scheme." - self.assertTrue(emsg in str(context.exception)) - emsg = "Variable mismatch in CCPPgt1_var_in_fort_meta_init, " + \ - "no Fortran variable bar." - self.assertTrue(emsg in str(context.exception)) - self.assertTrue("2 errors found comparing" in str(context.exception)) - - def test_ccpp_eq1_var_missing_in_meta(self): - """Test correct detection of a variable that - IS PRESENT the subroutine argument list - (due to a pre-processor directive: #ifdef CCPP), - and IS NOT PRESENT in meta file""" - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "CCPPeq1_var_missing_in_meta.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - _, _ = parse_scheme_files(scheme_files, self._run_env_ccpp) - # Verify 2 correct error messages returned - emsg = "Variable mismatch in CCPPeq1_var_missing_in_meta_finalize, "+ \ - "variables missing from metadata header." - self.assertTrue(emsg in str(context.exception)) - emsg = "Fortran variable, bar, not in metadata" - self.assertTrue(emsg in str(context.exception)) - self.assertTrue("2 errors found comparing" in str(context.exception)) - - def test_scheme_ddt_only(self): - """Test correct detection of a "scheme" file which contains only - DDT definitions""" - # Setup - scheme_files = [os.path.join(self._host_files_dir, "ddt2.meta")] - # Exercise - scheme_headers, table_dict = parse_scheme_files(scheme_files, - self._run_env_ccpp) - - def test_mismatch_hdim(self): - """Test correct use of loop variables (horizontal dimensions) - in scheme metadata. The allowed values depend on the phase - (run phase or not)""" - # Setup - scheme_files = [os.path.join(self._sample_files_dir, "mismatch_hdim.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - _, _ = parse_scheme_files(scheme_files, self._run_env_ccpp) - # Verify 2 correct error messages returned - emsg = "Invalid horizontal dimension, 'horizontal_dimension'" - self.assertTrue(emsg in str(context.exception)) - emsg = "Invalid horizontal dimension, 'horizontal_loop_extent'" - self.assertTrue(emsg in str(context.exception)) - -if __name__ == "__main__": - unittest.main() diff --git a/test/unit_tests/test_metadata_table.py b/test/unit_tests/test_metadata_table.py deleted file mode 100644 index 74f598d0..00000000 --- a/test/unit_tests/test_metadata_table.py +++ /dev/null @@ -1,445 +0,0 @@ -#! /usr/bin/env python3 -""" ------------------------------------------------------------------------ - Description: Contains unit tests for parse_metadata_file - in scripts file metadata_table.py - - Assumptions: - - Command line arguments: none - - Usage: python3 test_metadata_table.py # run the unit tests ------------------------------------------------------------------------ -""" -import sys -import os -import unittest - -UNIT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DIR = os.path.abspath(os.path.join(UNIT_TEST_DIR, os.pardir)) -SCRIPTS_DIR = os.path.abspath(os.path.join(TEST_DIR, os.pardir, "scripts")) -SAMPLE_FILES_DIR = os.path.join(UNIT_TEST_DIR, "sample_files") - -if not os.path.exists(SCRIPTS_DIR): - raise ImportError("Cannot find scripts directory") - -sys.path.append(SCRIPTS_DIR) - -# pylint: disable=wrong-import-position -from metadata_table import parse_metadata_file, MetadataTable -from framework_env import CCPPFrameworkEnv -# pylint: enable=wrong-import-position - -class MetadataTableTestCase(unittest.TestCase): - - """Tests for `parse_metadata_file`.""" - - _DUMMY_RUN_ENV = CCPPFrameworkEnv(None, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - - def test_good_host_file(self): - """Test that good host file test_host.meta returns one header named test_host""" - #Setup - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_host.meta") - #Exercise - result = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - #Verify that: - # no dependencies is returned as '' - # rel_path is returned as None - # size of returned list equals number of headers in the test file - # ccpp-table-properties name is 'test_host' - dependencies = result[0].dependencies - rel_path = result[0].dependencies_path - self.assertFalse('' in dependencies) - self.assertEqual(len(dependencies), 0) - self.assertIsNone(rel_path) - self.assertEqual(len(result), 1) - titles = [elem.table_name for elem in result] - self.assertIn('test_host', titles, msg="Header name 'test_host' is expected but not found") - - def test_good_multi_ccpp_arg_table(self): - """Test that good file with 4 ccpp-arg-table returns 4 headers""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_multi_ccpp_arg_tables.meta") - #Exercise - result = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - #Verify that size of returned list equals number of ccpp-table-properties in the test file - # ccpp-arg-tables are returned in result[0].sections() and result[1].sections() - self.assertEqual(len(result), 2) - - titles = list() - for table in result: - titles.extend([x.title for x in table.sections()]) - - self.assertIn('vmr_type', titles, msg="Header name 'vmr_type' is expected but not found") - self.assertIn('make_ddt_run', titles, msg="Header name 'make_ddt_run' is expected but not found") - self.assertIn('make_ddt_init', titles, msg="Header name 'make_ddt_init' is expected but not found") - self.assertIn('make_ddt_finalize', titles, msg="Header name 'make_ddt_finalize' is expected but not found") - - def test_bad_type_name(self): - """Test that `type = banana` returns expected error""" - #Setup - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_type_name.meta") - - #Exercise - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #Verify - #print("The exception is", context.exception) - self.assertTrue("Section type, 'banana', does not match table type, 'scheme'" in str(context.exception)) - - def test_double_header(self): - """Test that a duplicate header returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "double_header.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - self.assertTrue('table already contains \'test_host\'' in str(context.exception)) - - def test_bad_dimension(self): - """Test that `dimension = banana` returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_dimension.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - self.assertTrue('Invalid \'dimensions\' property value, \'' in str(context.exception)) - - def test_duplicate_variable(self): - """Test that a duplicate variable returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "test_duplicate_variable.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - self.assertTrue('Invalid (duplicate) standard name in temp_calc_adjust_run, defined at ' in str(context.exception)) - - def test_invalid_intent(self): - """Test that an invalid intent returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_invalid_intent.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - self.assertTrue('Invalid \'intent\' property value, \'banana\', at ' in str(context.exception)) - - def test_missing_intent(self): - """Test that a missing intent returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_missing_intent.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Required property, 'intent', missing, at " - self.assertTrue(emsg in str(context.exception)) - - def test_missing_units(self): - """Test that a missing units attribute returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_missing_units.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Required property, 'units', missing, at" - self.assertTrue(emsg in str(context.exception)) - - def test_missing_table_type(self): - """Test that a missing table type returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "test_missing_table_type.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Invalid section type, 'None'" - self.assertTrue(emsg in str(context.exception)) - - def test_bad_table_type(self): - """Test that a mismatched table type returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_table_type.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Section type, 'host', does not match table type, 'scheme'" - self.assertTrue(emsg in str(context.exception)) - - def test_missing_table_name(self): - """Test that a missing table name returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "test_missing_table_name.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Section name, 'None', does not match table title, 'test_missing_table_name'" - self.assertTrue(emsg in str(context.exception)) - - def test_bad_table_key(self): - """Test that a bad table key returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_table_key.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Invalid metadata table start property, 'banana', at " - self.assertTrue(emsg in str(context.exception)) - - def test_bad_line_split(self): - """Test that a bad split line with | returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_line_split.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Invalid variable property syntax, \'\', at " - self.assertTrue(emsg in str(context.exception)) - - def test_unknown_ddt_type(self): - """Test that a DDT type = banana returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_unknown_ddt_type.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Unknown DDT type, banana, at " - self.assertTrue(emsg in str(context.exception)) - - def test_bad_var_property_name(self): - """Test that a ddt_type = None returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_var_property_name.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Invalid variable property name, 'none', at " - self.assertTrue(emsg in str(context.exception)) - - def test_no_input(self): - """Test that no input returns expected error""" - with self.assertRaises(Exception) as context: - MetadataTable(self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "MetadataTable requires a name" - self.assertTrue(emsg in str(context.exception)) - - def test_no_table_type(self): - """Test that __init__ with table_type_in=None returns expected error""" - with self.assertRaises(Exception) as context: - MetadataTable(self._DUMMY_RUN_ENV, table_name_in="something", - table_type_in=None, dependencies=None, - dependencies_path=None, known_ddts=None, var_dict=None, - module=None, parse_object=None) - - #print("The exception is", context.exception) - emsg = "MetadataTable requires a table type" - self.assertTrue(emsg in str(context.exception)) - - def test_bad_header_type(self): - """Test that __init__ with table_type_in=banana returns expected error""" - with self.assertRaises(Exception) as context: - MetadataTable(self._DUMMY_RUN_ENV, table_name_in="something", - table_type_in="banana", dependencies=None, - dependencies_path=None, known_ddts=None, var_dict=None, - module=None, parse_object=None) - - #print("The exception is", context.exception) - emsg = "Invalid metadata arg table type, 'banana'" - self.assertTrue(emsg in str(context.exception)) - - def test_no_module(self): - """Test that __init__ with module=None returns expected error""" - with self.assertRaises(Exception) as context: - MetadataTable(self._DUMMY_RUN_ENV, table_name_in=None, - table_type_in=None, dependencies=None, - dependencies_path=None, known_ddts=None, var_dict=None, - module=None, parse_object=None) - - #print("The exception is", context.exception) - emsg = "MetadataTable requires a name" - self.assertTrue(emsg in str(context.exception)) - - def test_bad_1st_ccpp_arg_table(self): - """Test that first arg table named ccpp-farg-table returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "test_bad_1st_arg_table_header.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Invalid variable property syntax, '[ccpp-farg-table]', at " - self.assertTrue(emsg in str(context.exception)) - - def test_bad_2nd_ccpp_arg_table(self): - """Test that second arg table named ccpp-farg-table returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "test_bad_2nd_arg_table_header.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Invalid variable property syntax, '[ccpp-farg-table]', at " - self.assertTrue(emsg in str(context.exception)) - - def test_mismatch_section_table_title(self): - """Test that mismatched section name and table title - returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "test_mismatch_section_table_title.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Section name, 'test_host', does not match table title, 'banana', at " - self.assertTrue(emsg in str(context.exception)) - - def test_double_table_properties(self): - """Test that duplicate ccpp-table-properties returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "double_table_properties.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Duplicate metadata table, test_host, at " - self.assertTrue(emsg in str(context.exception)) - - def test_missing_table_properties(self): - """Test that a missing ccpp-table-properties returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "missing_table_properties.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Invalid CCPP metadata line, '[ccpp-arg-table]', at " - self.assertTrue(emsg in str(context.exception)) - - def test_dependencies_path(self): - """Test that dependencies_path and dependencies from ccpp-table-properties are read in correctly""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "test_dependencies_path.meta") - - result = parse_metadata_file(filename, known_ddts, - self._DUMMY_RUN_ENV) - - dependencies = result[0].dependencies - rel_path = result[0].dependencies_path - titles = [elem.table_name for elem in result] - - self.assertEqual(len(dependencies), 4) - phys_dir = os.path.join(TEST_DIR, "ccpp", "physics", "physics") - self.assertIn(os.path.join(phys_dir, 'machine.F'), dependencies, \ - msg="Dependency 'machine.F' is expected but not found") - self.assertIn(os.path.join(phys_dir, 'physcons.F90'), dependencies, \ - msg="Dependency 'physcons.F90' is expected but not found") - self.assertIn(os.path.join(phys_dir, 'GFDL_parse_tracers.F90'), dependencies, \ - msg="Dependency 'GFDL_parse_tracers.F90' is expected but not found") - self.assertIn(os.path.join(phys_dir, 'rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90'), dependencies, \ - msg="Header name 'rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90' is expected but not found") - - self.assertIn(rel_path, "../../ccpp/physics/physics") - self.assertEqual(len(result), 1) - self.assertIn('test_host', titles, msg="Table name 'test_host' is expected but not found") - - def test_invalid_table_properties_type(self): - """Test that an invalid ccpp-table-properties type returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "test_invalid_table_properties_type.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Invalid metadata table type, 'banana', at " - self.assertTrue(emsg in str(context.exception)) - - def test_added_kind_spec(self): - """Test that adding a kind_spec to a Metadata table works as expected""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "good_kind_spec_table_properties.meta") - - # Test that we can parse the table with no error - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - # Test that the expected names are in the table - kind_types = self._DUMMY_RUN_ENV.kind_types() - self.assertEqual(len(kind_types), 3) - self.assertIn("kind_temp", kind_types) - self.assertEqual(self._DUMMY_RUN_ENV.kind_module("kind_temp"), "fmodule") - self.assertEqual(self._DUMMY_RUN_ENV.kind_spec("kind_temp"), "temp_r8") - self.assertIn("temp_i8", kind_types) - self.assertIn("kind_phys", kind_types) - - def test_bad_kind_spec(self): - """Test that adding a bad kind_spec to a Metadata table returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "bad_kind_spec_table_properties.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - emsg = "A Fortran kind name is required for 'temp_r8'" - self.assertTrue(emsg in str(context.exception), msg=str(context.exception)) - - def test_duplicate_kind_spec(self): - """Test that adding a duplicate kind_spec to a Metadata table returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "duplicate_kind_spec_table_properties.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - emsg = ("'kind_temp = [kind_temp, fmodule]'is an invalid duplicate. " - "kind_temp is already '['temp_r8', 'fmodule'],") - self.assertTrue(emsg in str(context.exception), msg=str(context.exception)) - -if __name__ == "__main__": - unittest.main() diff --git a/test/unit_tests/test_sdf.py b/test/unit_tests/test_sdf.py deleted file mode 100644 index fbb2c1db..00000000 --- a/test/unit_tests/test_sdf.py +++ /dev/null @@ -1,569 +0,0 @@ -#! /usr/bin/env python3 -""" ------------------------------------------------------------------------ - Description: Contains unit tests for parsing Suite Definition Files (SDFs) - in scripts/parse_tools/xml_tools.py - - Assumptions: - - Command line arguments: none - - Usage: python3 test_sdf.py # run the unit tests ------------------------------------------------------------------------ -""" - -import filecmp -import glob -import logging -import os -import sys -import unittest -import xml.etree.ElementTree as ET - -_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -_SCRIPTS_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, - os.pardir, "scripts")) -_SAMPLE_FILES_DIR = os.path.join(_TEST_DIR, "sample_suite_files") -_PRE_TMP_DIR = os.path.join(_TEST_DIR, "tmp") -_TMP_DIR = os.path.join(_PRE_TMP_DIR, "suite_files") - -if not os.path.exists(_SCRIPTS_DIR): - raise ImportError(f"Cannot find scripts directory, {_SCRIPTS_DIR}") - -sys.path.append(_SCRIPTS_DIR) - -# pylint: disable=wrong-import-position -from parse_tools import init_log -from parse_tools import read_xml_file, validate_xml_file, write_xml_file -from parse_tools import find_schema_version, expand_nested_suites -# pylint: enable=wrong-import-position - -class SDFParseTestCase(unittest.TestCase): - - """Tests for `expand_nested_suites` and related functions.""" - - logger = None - - @classmethod - def setUpClass(cls): - """Clean output directory (tmp) before running tests""" - # Does "tmp" directory exist? If not then create it: - if not os.path.exists(_PRE_TMP_DIR): - os.makedirs(_PRE_TMP_DIR) - # end if - - # We need a logger - cls.logger = init_log(cls.__name__, level=logging.WARNING) - - #Does "tmp" directory exist? If not then create it: - # Ensure the "tmp/suite_files" directory exists and is empty - if os.path.exists(_TMP_DIR): - # Clear out all files: - for fpath in glob.iglob(os.path.join(_TMP_DIR, '*.*')): - if os.path.exists(fpath): - os.remove(fpath) - # End if - # End for - else: - os.makedirs(_TMP_DIR) - # end if - - # Run inherited setup method: - super().setUpClass() - - @classmethod - def get_logger(cls): - return cls.logger - - @classmethod - def compare_text(cls, name, txt1, txt2, typ): - """Compare two XML text or tail items (which may be None). - Return None if items match, otherwise, return an error string""" - res = None - if txt1 and txt2: - if txt1.strip() != txt2.strip(): - res = f"{name} {typ}, '{txt1}', does not match {typ}, '{txt2}'" - # end if - elif txt1: - res = f"{name} {typ} is missing from string2" - elif txt2: - res = f"{name} {typ} is missing from string1" - else: - res = None - # end if - return res - - @classmethod - def xml_diff(cls, xt1, xt2): - """ - Compares two xml etrees, xt1 and xt2 - Return None if the trees match, otherwise, return a difference string - """ - - diffs = [] - # First, compare the XML tags - if xt1.tag != xt2.tag: - diffs.append(f"Tags do not match: {xt1.tag} != {xt2.tag}") - else: - # Compare the attributes - for name, value in xt1.attrib.items(): - if name not in xt2.attrib: - diffs.append(f"xt1 attribute, {name}, is missing in xt2") - else: - xt2v = xt2.attrib.get(name) - if xt2v != value: - diffs.append(f"Attributes for {name} do not match: {str(value)} != {str(xt2v)}") - # end if - # end if - # end for - for name in xt2.attrib.keys(): - if name not in xt1.attrib: - diffs.append(f"xt2 attribute, {name}, is missing in xt1") - # end if - # end for - # Compare the text bodies (if any) - tdiff = cls.compare_text(xt1.tag, xt1.text, xt2.text, "text") - if tdiff: - diffs.append(tdiff) - # end if - tdiff = cls.compare_text(xt1.tag, xt1.tail, xt2.tail, "tail") - if tdiff: - diffs.append(tdiff) - # end if - # Compare children - if len(xt1) != len(xt2): - diffs.append(f"Number of children length differs, {len(xt1)} != {len(xt2)}") - else: - for child1, child2 in zip(xt1, xt2): - kid_diffs = cls.xml_diff(child1, child2) - if kid_diffs: - diffs.extend(kid_diffs) - # end if - # end for - # end if - # end if - return diffs - - def test_xml_diff(self): - """Test that xml_diff catches xml differences""" - root1 = ET.fromstring("item") - root2 = ET.fromstring("item") - diffs = self.xml_diff(root1, root2) - self.assertTrue(diffs) - self.assertEqual(len(diffs), 1) - self.assertTrue("Tags do not match" in diffs[0], - msg="tag1 should not match taga") - root1 = ET.fromstring("item1") - root2 = ET.fromstring("item2") - diffs = self.xml_diff(root1, root2) - self.assertTrue(diffs) - self.assertEqual(len(diffs), 1) - self.assertTrue("does not match" in diffs[0], - msg="item1 should not match item2") - root1 = ET.fromstring('item1') - root2 = ET.fromstring('item1') - diffs = self.xml_diff(root1, root2) - self.assertTrue(diffs) - self.assertEqual(len(diffs), 3) - self.assertTrue("Attributes for" in diffs[0] and "do not match" in diffs[0], - msg="attrib1 values should not match") - self.assertTrue("xt1 attribute, attrib2, is missing in xt2" in diffs[1], - msg=f"attrib2 is missing in root2") - self.assertTrue("xt2 attribute, attrib3, is missing in xt1" in diffs[2], - msg=f"attrib3 is missing in root1") - root1 = ET.fromstring('') - root2 = ET.fromstring('') - diffs = self.xml_diff(root1, root2) - self.assertEqual(len(diffs), 1) - self.assertTrue("Attributes for" in diffs[0] and "do not match" in diffs[0], - msg=f"attrib2 values should not match") - - def test_good_v1_sdf(self): - """Test that the parser recognizes a V1 SDF and parses it correctly - """ - num_tests = 2 - header = "Test of parsing of good V1 SDF" - for test_num in range(num_tests): - # Setup - testname = f"suite_good_v1_test{test_num+1:{0}{2}}" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 1) - self.assertEqual(schema_version[1], 0) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res) - write_xml_file(xml_root, compare, logger) - amsg = f"{compare} does not exist" - self.assertTrue(os.path.exists(compare), msg=amsg) - _, compare_root = read_xml_file(compare, logger) - diffs = self.xml_diff(xml_root, compare_root) - lsep = '\n' - amsg = f"{source} does not match {compare}\n{lsep.join(diffs)}" - self.assertFalse(diffs, msg=amsg) - # end for - - def test_good_v2_sdf_01(self): - """Test that the parser recognizes a V2 SDF and parses and - expands it correctly. - Test the expansion of one group of a simple nested suite at group level. - """ - header = "Test of parsing of good V2 SDF" - # Setup - testname = "suite_good_v2_test01" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - source_exp = os.path.join(_SAMPLE_FILES_DIR, f"{testname}_exp.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) - write_xml_file(xml_root, compare, logger) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res) - amsg = f"{compare} does not exist" - self.assertTrue(os.path.exists(compare), msg=amsg) - _, xml_root = read_xml_file(source_exp, logger) - _, compare_root = read_xml_file(compare, logger) - diffs = self.xml_diff(xml_root, compare_root) - lsep = '\n' - amsg = f"{source_exp} does not match {compare}\n{lsep.join(diffs)}" - self.assertFalse(diffs, msg=amsg) - - def test_good_v2_sdf_02(self): - """Test that the parser recognizes a V2 SDF and parses and - expands it correctly - Test the expansion of one group of a multiple group nested suite at group level. - """ - header = "Test of parsing of good V2 SDF" - # Setup - testname = "suite_good_v2_test02" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - source_exp = os.path.join(_SAMPLE_FILES_DIR, f"{testname}_exp.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) - write_xml_file(xml_root, compare, logger) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res) - amsg = f"{compare} does not exist" - self.assertTrue(os.path.exists(compare), msg=amsg) - _, xml_root = read_xml_file(source_exp, logger) - _, compare_root = read_xml_file(compare, logger) - diffs = self.xml_diff(xml_root, compare_root) - lsep = '\n' - amsg = f"{source_exp} does not match {compare}\n{lsep.join(diffs)}" - self.assertFalse(diffs, msg=amsg) - - def test_good_v2_sdf_03(self): - """Test that the parser recognizes a V2 SDF and parses and - expands it correctly - Test expansion of two nested suites at group level plus a full nested suite at - suite level. - """ - header = "Test of parsing of good V2 SDF" - # Setup - testname = "suite_good_v2_test03" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - source_exp = os.path.join(_SAMPLE_FILES_DIR, f"{testname}_exp.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) - write_xml_file(xml_root, compare, logger) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res) - amsg = f"{compare} does not exist" - self.assertTrue(os.path.exists(compare), msg=amsg) - _, xml_root = read_xml_file(source_exp, logger) - _, compare_root = read_xml_file(compare, logger) - diffs = self.xml_diff(xml_root, compare_root) - lsep = '\n' - amsg = f"{source_exp} does not match {compare}\n{lsep.join(diffs)}" - self.assertFalse(diffs, msg=amsg) - - def test_good_v2_sdf_04(self): - """Test that the parser recognizes a V2 SDF and parses and - expands it correctly - Test expansion of two nested suites at group level plus one group from a - nested suite at suite level. - """ - header = "Test of parsing of good V2 SDF" - # Setup - testname = "suite_good_v2_test04" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - source_exp = os.path.join(_SAMPLE_FILES_DIR, f"{testname}_exp.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) - write_xml_file(xml_root, compare, logger) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res) - amsg = f"{compare} does not exist" - self.assertTrue(os.path.exists(compare), msg=amsg) - _, xml_root = read_xml_file(source_exp, logger) - _, compare_root = read_xml_file(compare, logger) - diffs = self.xml_diff(xml_root, compare_root) - lsep = '\n' - amsg = f"{source_exp} does not match {compare}\n{lsep.join(diffs)}" - self.assertFalse(diffs, msg=amsg) - - def test_bad_v2_suite_tag_sdf(self): - """Test that verification system recognizes a misplaced suite tag""" - header = "Test trapping of version attribute on a v2 suite tag" - # Setup - testname = f"suite_bad_v2_suite_tag" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - # Some versions of xmllint return an exit code 0 even if the - # validation fails. "Good" versions return an exit code /= 0, - # which then raises a CCPPError internally. The following - # logic handles the correct behavior (validation fails ==> - # exit code /= 0 ==> CCPPError). - try: - res = validate_xml_file(source, 'suite', schema_version, logger) - except Exception as e: - emsg = "Schemas validity error : Element 'suite': This element is not expected." - msg = str(e) - self.assertTrue(emsg in msg) - - def test_bad_v2_suite_duplicate_group1(self): - """Test that verification system recognizes a duplicate group name""" - header = "Test trapping of expanded suite duplicate group name" - # Setup - testname = f"suite_bad_v2_duplicate_group" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res, msg="Initial suite file should be valid") - with self.assertRaises(Exception) as context: - expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) - write_xml_file(xml_root, compare, logger) - _ = validate_xml_file(compare, 'suite', schema_version, logger) - # end with - emsg = "Schemas validity error : Element 'group', attribute 'name': " + \ - "'group1' is not a valid value of the atomic type 'fortran_id_type_unique'" - fmsg = str(context.exception) - self.assertTrue(emsg in fmsg, msg=fmsg) - if not emsg in fmsg: - raise context - - def test_bad_v2_suite_missing_group(self): - """Test that verification system recognizes a missing group name""" - header = "Test trapping of expanded suite missing group name" - # Setup - testname = f"suite_missing_group" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res, msg="Initial suite file should be valid") - with self.assertRaises(Exception) as context: - expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) - write_xml_file(xml_root, compare, logger) - # end with - emsg = "Nested suite subsuite_1, group group2, not found" - fmsg = str(context.exception) - self.assertTrue(emsg in fmsg, msg=fmsg) - - def test_bad_v2_suite_missing_file(self): - """Test that verification system recognizes a missing file argument""" - header = "Test trapping of missing file for nested suite" - # Setup - testname = f"suite_missing_file" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - # See note about different behavior of xmllint versions - # in test test_bad_v2_suite_tag_sdf above. - try: - res = validate_xml_file(source, 'suite', schema_version, logger) - except Exception as e: - emsg = "Schemas validity error : Element 'nested_suite': " + \ - "The attribute 'file' is required but missing." - msg = str(e) - self.assertTrue(emsg in msg) - - def test_bad_v2_suite_missing_loaded_suite(self): - """Test that verification system recognizes a missing suite loaded - from another file""" - header = "Test trapping of expanded suite missing a subsuite in a different file" - # Setup - testname = f"suite_missing_loaded_suite" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res, msg="Initial suite file should be valid") - with self.assertRaises(Exception) as context: - expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) - write_xml_file(xml_root, compare, logger) - # end with - emsg = "Nested suite v12_suite, group main_group, not found in file" - fmsg = str(context.exception) - self.assertTrue(emsg in fmsg, msg=fmsg) - - def test_bad_v2_suite_infinite_group_recursion(self): - """Test that verification system recognizes infinite recursion when - including at the group level""" - header = "Test trapping of expanded suite with infinite group recursion" - # Setup - testname = f"suite_recurse_top1" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res, msg="Initial suite file should be valid") - with self.assertRaises(Exception) as context: - expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) - write_xml_file(xml_root, compare, logger) - # end with - emsg = ("Exceeded number of iterations while expanding nested suites") - fmsg = str(context.exception) - self.assertTrue(emsg in fmsg, msg=fmsg) - - def test_bad_v2_suite_infinite_suite_recursion(self): - """Test that verification system recognizes infinite recursion when - including at the imported suite level""" - header = "Test trapping of expanded suite with infinite suite recursion" - # Setup - testname = f"suite_recurse_top2" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res, msg="Initial suite file should be valid") - with self.assertRaises(Exception) as context: - expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) - write_xml_file(xml_root, compare, logger) - # end with - emsg = ("Exceeded number of iterations while expanding nested suites") - fmsg = str(context.exception) - self.assertTrue(emsg in fmsg, msg=fmsg) - - def test_bad_schema_version(self): - """Test that verification system recognizes a bad version entry""" - num_tests = 4 - header = "Test trapping of invalid SDF version" - exc_strings = ["Format must be .", - "Format must be .", - "Major version must be at least 1", - "Minor version must be non-negative"] - for test_num in range(num_tests): - # Setup - testname = f"suite_bad_version{test_num+1:{0}{2}}" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - logger = self.get_logger() - # Exercise - with self.assertRaises(Exception) as context: - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - # end with - # Check exception for expected error messages - exp_str = str(context.exception) - self.assertTrue(exc_strings[test_num] in exp_str, - msg=f"Bad exception in test {test_num + 1}, '{exp_str}'") - # end for - - def test_missing_schema_version(self): - """Test that verification system recognizes a missing version num""" - header = "Test trapping of missing SDF version" - # Setup - testname = f"suite_missing_version" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - logger = self.get_logger() - # Exercise - with self.assertRaises(Exception) as context: - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - # end with - # Check exception for expected error messages - self.assertTrue("version attribute required" in str(context.exception), - msg=f"Bad exception for missing suite version") - - def test_invalid_fortran_id(self): - """Test that verification system recognizes a bad Fortran ID entry""" - num_tests = 3 - header = "Test trapping of invalid Fortran ID" - tests = [ - "suite_invalid_scheme_fortran_id", - "suite_invalid_group_fortran_id", - "suite_invalid_suite_fortran_id", - ] - exc_strings = [ - "The value 'scheme-1' is not accepted by the pattern '[A-Za-z][A-Za-z0-9_]{0,63}", - "The value 'group-1' is not accepted by the pattern '[A-Za-z][A-Za-z0-9_]{0,63}", - "The value 'ver-test-suite' is not accepted by the pattern '[A-Za-z][A-Za-z0-9_]{0,63}", - ] - for test_num in range(num_tests): - # Setup - testname = tests[test_num] - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - logger = self.get_logger() - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - # Exercise - with self.assertRaises(Exception) as context: - res = validate_xml_file(source, 'suite', schema_version, logger) - # end with - # Check exception for expected error messages - exp_str = str(context.exception) - self.assertTrue(exc_strings[test_num] in exp_str, - msg=f"Bad exception in test {test_num + 1}, '{exp_str}'") - # end for diff --git a/test/unit_tests/test_var_transforms.py b/test/unit_tests/test_var_transforms.py deleted file mode 100644 index 3d5e3477..00000000 --- a/test/unit_tests/test_var_transforms.py +++ /dev/null @@ -1,475 +0,0 @@ -#! /usr/bin/env python3 -""" ------------------------------------------------------------------------ - Description: Contains unit tests for variable transforms involving - a VarCompatObj object - - Assumptions: - - Command line arguments: none - - Usage: python test_var_transforms.py # run the unit tests ------------------------------------------------------------------------ -""" -import sys -import os -import unittest - -TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -SCRIPTS_DIR = os.path.abspath(os.path.join(TEST_DIR, os.pardir, os.pardir, "scripts")) -SAMPLE_FILES_DIR = os.path.join(TEST_DIR, "sample_files") - -if not os.path.exists(SCRIPTS_DIR): - raise ImportError("Cannot find scripts directory") - -sys.path.append(SCRIPTS_DIR) - -# pylint: disable=wrong-import-position -from framework_env import CCPPFrameworkEnv -from metavar import Var -from parse_tools import ParseContext, ParseSource, ParseSyntaxError -from var_props import VarCompatObj -# pylint: enable=wrong-import-position - -class VarCompatTestCase(unittest.TestCase): - - """Tests for variable transforms.""" - - def _new_var(self, standard_name, units, dimensions, vtype, vkind=''): - """Create and return a new Var object with the requested properties""" - context = ParseContext(linenum=self.__linenum, filename="foo.meta") - source = ParseSource("foo", "host", context) - prop_dict = {'local_name' : f"foo{self.__linenum}", - 'standard_name' : standard_name, - 'units' : units, - 'dimensions' : f"({', '.join(dimensions)})", - 'type' : vtype, 'kind' : vkind} - self.__linenum += 5 - return Var(prop_dict, source, self.__run_env) - - def setUp(self): - """Setup variables for testing""" - self.__run_env = CCPPFrameworkEnv(None, ndict={'host_files':'', - 'scheme_files':'foo.meta', - 'suites':''}, - kind_types=["kind_phys=REAL64", - "kind_dyn=REAL32", - "kind_host=REAL64"]) - # For making variables unique - self.__linenum = 2 - # For assert messages - self.__inst_emsg = "Var.compatible returned a '{}', not a VarCompatObj" - - def test_equiv_vars(self): - """Test that equivalent variables are reported as equivalent""" - int_scalar1 = self._new_var('int_stdname1', 'm s-1', [], 'integer') - int_array1 = self._new_var('int_stdname2', 'm s-1', ['hdim'], - 'real', vkind='kind_phys') - int_array2 = self._new_var('int_stdname2', 'm s-1', ['hdim'], - 'real', vkind='kind_host') - int_array3 = self._new_var('int_stdname2', 'm s-1', ['hdim'], - 'real', vkind='REAL64') - compat = int_scalar1.compatible(int_scalar1, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertTrue(compat) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertFalse(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertFalse(compat.has_unit_transforms) - compat = int_array1.compatible(int_array2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertTrue(compat) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertFalse(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertFalse(compat.has_unit_transforms) - compat = int_array3.compatible(int_array2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertTrue(compat) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertFalse(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertFalse(compat.has_unit_transforms) - - def test_incompatible_vars(self): - """Test that incompatible variables are reported correctly""" - int_scalar1 = self._new_var('int_stdname1', 'm s-1', [], 'integer') - int_scalar2 = self._new_var('int_stdname2', 'm s-1', [], 'integer') - int_array1 = self._new_var('int_stdname1', 'm s-1', ['hdim'], - 'integer') - real_array1 = self._new_var('int_stdname1', 'm s-1', ['hdim'], - 'real', vkind='kind_phys') - # Array and scalar - compat = int_scalar1.compatible(int_array1, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertFalse(compat) - self.assertFalse(compat.compat) - self.assertEqual(compat.incompat_reason, 'dimensions') - # Variables with different standard names - compat = int_scalar1.compatible(int_scalar2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertFalse(compat) - self.assertFalse(compat.compat) - self.assertEqual(compat.incompat_reason, 'standard names') - # Variables with different types - compat = int_array1.compatible(real_array1, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertFalse(compat) - self.assertFalse(compat.compat) - self.assertEqual(compat.incompat_reason, 'types') - - def test_valid_unit_change(self): - """Test that valid unit changes are detected""" - real_scalar1 = self._new_var('real_stdname1', 'm', [], - 'real', vkind='kind_phys') - real_scalar2 = self._new_var('real_stdname1', 'mm', [], - 'real', vkind='kind_phys') - compat = real_scalar1.compatible(real_scalar2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertFalse(compat.equiv) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertFalse(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertTrue(compat.has_unit_transforms) - - real_array1 = self._new_var('real_stdname1', 'm s-1', ['hdim', 'vdim'], - 'real', vkind='kind_phys') - real_array2 = self._new_var('real_stdname1', 'km h-1', ['hdim', 'vdim'], - 'real', vkind='kind_phys') - compat = real_scalar1.compatible(real_scalar2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertFalse(compat.equiv) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertFalse(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertTrue(compat.has_unit_transforms) - - def test_unsupported_unit_change(self): - """Test that unsupported unit changes are detected""" - real_scalar1 = self._new_var('real_stdname1', 'min', [], - 'real', vkind='kind_phys') - real_scalar2 = self._new_var('real_stdname1', 'd', [], - 'real', vkind='kind_phys') - char_nounit1 = self._new_var('char_stdname1', 'none', [], - 'character', vkind='len=256') - char_nounit2 = self._new_var('char_stdname1', '1', [], - 'character', vkind='len=256') - with self.assertRaises(ParseSyntaxError) as context: - compat = real_scalar1.compatible(real_scalar2, self.__run_env) - # end with - #Test bad conversion for real time variables - #Verify correct error message returned - emsg = "Unsupported unit conversion, 'min' to 'd' for 'real_stdname1'" - self.assertTrue(emsg in str(context.exception)) - #Test bad conversion for unitless variables - with self.assertRaises(ParseSyntaxError) as context: - compat = char_nounit1.compatible(char_nounit2, self.__run_env) - # end with - #Verify correct error message returned - emsg = "Unsupported unit conversion, 'none' to '1' for 'char_stdname1'" - self.assertTrue(emsg in str(context.exception)) - - def test_valid_kind_change(self): - """Test that valid kind changes are detected""" - real_scalar1 = self._new_var('real_stdname1', 'mm', [], - 'real', vkind='kind_phys') - real_scalar2 = self._new_var('real_stdname1', 'mm', [], - 'real', vkind='kind_dyn') - compat = real_scalar1.compatible(real_scalar2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertFalse(compat) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertTrue(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertFalse(compat.has_unit_transforms) - - real_scalar1 = self._new_var('real_stdname1', 'm', [], - 'real', vkind='kind_phys') - real_scalar2 = self._new_var('real_stdname1', 'mm', [], - 'real', vkind='REAL32') - compat = real_scalar1.compatible(real_scalar2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertFalse(compat) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertTrue(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertTrue(compat.has_unit_transforms) - - def test_valid_dim_change(self): - """Test that valid dimension changes are detected""" - real_array1 = self._new_var('real_stdname1', 'C', - ['horizontal_dimension', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - real_array2 = self._new_var('real_stdname1', 'K', - ['ccpp_constant_one:horizontal_loop_extent', - 'vertical_layer_dimension'], - 'real', vkind='kind_dyn') - compat = real_array1.compatible(real_array2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertFalse(compat) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertTrue(compat.has_kind_transforms) - self.assertTrue(compat.has_dim_transforms) - self.assertTrue(compat.has_unit_transforms) - - real_array1 = self._new_var('real_stdname1', 'C', - ['ccpp_constant_one:horizontal_dimension', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - real_array2 = self._new_var('real_stdname1', 'K', - ['vertical_layer_dimension', - 'horizontal_loop_extent'], - 'real', vkind='kind_dyn') - compat = real_array1.compatible(real_array2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertFalse(compat) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertTrue(compat.has_kind_transforms) - self.assertTrue(compat.has_dim_transforms) - self.assertTrue(compat.has_unit_transforms) - - def test_valid_dim_transforms(self): - """Test that valid variable transform code is created""" - real_array1 = self._new_var('real_stdname1', 'C', - ['horizontal_dimension', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - real_array2 = self._new_var('real_stdname1', 'C', - ['ccpp_constant_one:horizontal_loop_extent', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - real_array3 = self._new_var('real_stdname1', 'K', - ['ccpp_constant_one:horizontal_loop_extent', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - real_array4 = self._new_var('real_stdname1', 'K', - ['ccpp_constant_one:horizontal_loop_extent', - 'vertical_layer_dimension'], - 'real', vkind='kind_dyn') - real_array5 = self._new_var('real_stdname1', 'K', - ['vertical_layer_dimension', - 'ccpp_constant_one:horizontal_dimension'], - 'real', vkind='kind_phys') - v1_lname = real_array1.get_prop_value('local_name') - v2_lname = real_array2.get_prop_value('local_name') - v3_lname = real_array3.get_prop_value('local_name') - v4_lname = real_array4.get_prop_value('local_name') - v5_lname = real_array5.get_prop_value('local_name') - # Comparison between equivalent variables - compat = real_array1.compatible(real_array2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - rindices = ("hind", "vind") - lindices = rindices - fwd_stmt = compat.forward_transform(v2_lname, v1_lname, rindices, lindices) - ind_str = ','.join(rindices) - expected = f"{v2_lname}({ind_str}) = {v1_lname}({ind_str})" - self.assertEqual(fwd_stmt, expected) - rev_stmt = compat.reverse_transform(v1_lname, v2_lname, rindices, lindices) - expected = f"{v1_lname}({ind_str}) = {v2_lname}({ind_str})" - self.assertEqual(rev_stmt, expected) - - # Comparison between equivalent variables with loop correction - compat = real_array1.compatible(real_array2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - rindices = ("hind", "vind") - lindices = ("hind-col_start+1", "vind") - fwd_stmt = compat.forward_transform(v2_lname, v1_lname, rindices, lindices) - lind_str = ','.join(lindices) - rind_str = ','.join(rindices) - expected = f"{v2_lname}({lind_str}) = {v1_lname}({rind_str})" - self.assertEqual(fwd_stmt, expected) - lindices = ("hind+col_start-1", "vind") - rev_stmt = compat.reverse_transform(v1_lname, v2_lname, rindices, lindices) - lind_str = ','.join(lindices) - expected = f"{v1_lname}({lind_str}) = {v2_lname}({rind_str})" - self.assertEqual(rev_stmt, expected) - - # Comparison between equivalent variables with loop correction - # plus vertical flip - compat = real_array1.compatible(real_array2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - rindices = ("hind", "vind") - lindices = ("hind-col_start+1", "pver-vind+1") - fwd_stmt = compat.forward_transform(v2_lname, v1_lname, rindices, lindices) - lind_str = ','.join(lindices) - rind_str = ','.join(rindices) - expected = f"{v2_lname}({lind_str}) = {v1_lname}({rind_str})" - self.assertEqual(fwd_stmt, expected) - lindices = ("hind+col_start-1", "pver-vind+1") - rev_stmt = compat.reverse_transform(v1_lname, v2_lname, rindices, lindices) - lind_str = ','.join(lindices) - expected = f"{v1_lname}({lind_str}) = {v2_lname}({rind_str})" - self.assertEqual(rev_stmt, expected) - - # Comparison between variables with different units - compat = real_array1.compatible(real_array3, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - rindices = ("hind", "vind") - lindices = ("hind-col_start+1", "vind") - conv = f"273.15_{real_array1.get_prop_value('kind')}" - fwd_stmt = compat.forward_transform(v3_lname, v1_lname, rindices, lindices) - lind_str = ','.join(lindices) - rind_str = ','.join(rindices) - expected = f"{v3_lname}({lind_str}) = {v1_lname}({rind_str})+{conv}" - self.assertEqual(fwd_stmt, expected) - lindices = ("hind+col_start-1", "vind") - rev_stmt = compat.reverse_transform(v1_lname, v3_lname, rindices, lindices) - lind_str = ','.join(lindices) - conv = f"273.15_{real_array2.get_prop_value('kind')}" - expected = f"{v1_lname}({lind_str}) = {v3_lname}({rind_str})-{conv}" - self.assertEqual(rev_stmt, expected) - - # Comparison between variables with different kind - compat = real_array4.compatible(real_array3, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - rindices = ("hind", "vind") - lindices = ("hind", "vind") - fwd_stmt = compat.forward_transform(v4_lname, v3_lname, rindices, lindices) - lind_str = ','.join(lindices) - rind_str = ','.join(rindices) - rkind = real_array3.get_prop_value('kind') - expected = f"{v4_lname}({lind_str}) = real({v3_lname}({rind_str}), {rkind})" - self.assertEqual(fwd_stmt, expected) - lindices = ("hind", "vind") - rev_stmt = compat.reverse_transform(v3_lname, v4_lname, rindices, lindices) - lind_str = ','.join(lindices) - rkind = real_array4.get_prop_value('kind') - expected = f"{v3_lname}({lind_str}) = real({v4_lname}({rind_str}), {rkind})" - self.assertEqual(rev_stmt, expected) - - # Comparison between variables with different units and kind - compat = real_array1.compatible(real_array4, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - rindices = ("hind", "vind") - lindices = ("hind-col_start+1", "vind") - rkind = real_array4.get_prop_value('kind') - conv = f"273.15_{rkind}" - fwd_stmt = compat.forward_transform(v2_lname, v1_lname, rindices, lindices) - lind_str = ','.join(lindices) - rind_str = ','.join(rindices) - expected = f"{v2_lname}({lind_str}) = real({v1_lname}({rind_str}), {rkind})+{conv}" - self.assertEqual(fwd_stmt, expected) - lindices = ("hind+col_start-1", "vind") - rev_stmt = compat.reverse_transform(v1_lname, v2_lname, rindices, lindices) - lind_str = ','.join(lindices) - rkind = real_array1.get_prop_value('kind') - conv = f"273.15_{rkind}" - expected = f"{v1_lname}({lind_str}) = real({v2_lname}({rind_str}), {rkind})-{conv}" - self.assertEqual(rev_stmt, expected) - - # Comparison between variables with different dimension ordering - # and horizontal loop adjustment and vertical flip - compat = real_array5.compatible(real_array3, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - rindices = ("hind", "vind") - lindices = ("pver-vind+1", "hind-col_start+1") - fwd_stmt = compat.forward_transform(v4_lname, v5_lname, rindices, lindices) - lind_str = ','.join(lindices) - rind_str = ','.join(rindices) - rkind = real_array3.get_prop_value('kind') - expected = f"{v4_lname}({lind_str}) = {v5_lname}({rind_str})" - self.assertEqual(fwd_stmt, expected) - rindices = ("vind", "hind") - rind_str = ','.join(rindices) - lindices = ("hind+col_start-1", "pver-vind+1") - rev_stmt = compat.reverse_transform(v5_lname, v4_lname, rindices, lindices) - lind_str = ','.join(lindices) - rkind = real_array4.get_prop_value('kind') - expected = f"{v5_lname}({lind_str}) = {v4_lname}({rind_str})" - self.assertEqual(rev_stmt, expected) - - def test_compatible_tendency_variable(self): - """Test that a given tendency variable is compatible with - its corresponding state variable""" - real_array1 = self._new_var('real_stdname1', 'C', - ['horizontal_dimension', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - real_array2 = self._new_var('tendency_of_real_stdname1', 'C s-1', - ['horizontal_dimension', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - compat = real_array2.compatible(real_array1, self.__run_env, is_tend=True) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertTrue(compat) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertFalse(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertFalse(compat.has_unit_transforms) - - def test_compatible_tendency_variable_equivalent_units(self): - """Test that a given tendency variable is compatible with - its corresponding state variable""" - real_array1 = self._new_var('real_stdname1', 'V A', - ['horizontal_dimension', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - real_array2 = self._new_var('tendency_of_real_stdname1', 'W s-1', - ['horizontal_dimension', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - compat = real_array2.compatible(real_array1, self.__run_env, is_tend=True) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertTrue(compat) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertFalse(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertFalse(compat.has_unit_transforms) - - def test_incompatible_tendency_variable(self): - """Test that the correct error is returned when a given tendency - variable has inconsistent units vs the state variable""" - real_array1 = self._new_var('real_stdname1', 'm', - ['horizontal_dimension', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - real_array2 = self._new_var('tendency_of_real_stdname1', 'cm s-1', - ['horizontal_dimension', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - compat = real_array2.compatible(real_array1, self.__run_env, is_tend=True) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - # Verify correct error message returned - emsg = "\nMismatch tendency variable units 'cm s-1' for variable 'tendency_of_real_stdname1'. No variable transforms supported for tendencies. Tendency units should be 'm s-1' to match state variable." - self.assertEqual(compat.incompat_reason, emsg) - self.assertFalse(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertFalse(compat.has_unit_transforms) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/unit_tests/xmllint_wrapper/xmllint b/test/unit_tests/xmllint_wrapper/xmllint deleted file mode 100755 index 4f043c1c..00000000 --- a/test/unit_tests/xmllint_wrapper/xmllint +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python3 - -# This is a wrapper around xmllint to emulate the "bad behavior" -# of some xmllint versions that return an exit code 0 even if the -# validation fails. It requires the full path to the "real" xmllint -# executable to be defined as environment variable XMLLINT_REAL - -import os -import shutil -import subprocess -import sys - -xmllint = os.getenv("XMLLINT_REAL") -if not xmllint: - raise Exception("xmllint not found") - -cmd = [xmllint] + sys.argv[1:] -cproc = subprocess.run(cmd, check=False, capture_output=True) -if cproc.stdout: - sys.stdout.write(cproc.stdout.decode('utf-8', errors='replace').strip()+"\n") -if cproc.stderr: - sys.stderr.write(cproc.stderr.decode('utf-8', errors='replace').strip()+"\n") -# Exit with an exit code of zero no matter what -sys.exit(0) diff --git a/test/utils/CMakeLists.txt b/test/utils/CMakeLists.txt deleted file mode 100644 index dee888ca..00000000 --- a/test/utils/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_library(test_utils STATIC test_utils.F90) diff --git a/test/var_compatibility_test/CMakeLists.txt b/test/var_compatibility_test/CMakeLists.txt deleted file mode 100644 index 2938c2d0..00000000 --- a/test/var_compatibility_test/CMakeLists.txt +++ /dev/null @@ -1,49 +0,0 @@ - -#------------------------------------------------------------------------------ -# -# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES -# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) -# -#------------------------------------------------------------------------------ -set(SCHEME_FILES "effr_calc" "effrs_calc" "effr_diag" "effr_pre" "effr_post" "rad_lw" "rad_sw") -set(HOST_FILES "module_rad_ddt" "test_host_data" "test_host_mod") -set(SUITE_FILES "var_compatibility_suite.xml") -# HOST is the name of the executable we will build. -# We assume there are files ${HOST}.meta and ${HOST}.F90 in CMAKE_SOURCE_DIR -set(HOST "test_host") - -# By default, generated caps go in ccpp subdir -set(CCPP_CAP_FILES "${CMAKE_CURRENT_BINARY_DIR}/ccpp") - -# Create lists for Fortran and meta data files from file names -list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) -list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_META_FILES) -list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE VAR_COMPATIBILITY_HOST_FORTRAN_FILES) -list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE VAR_COMPATIBILITY_HOST_METADATA_FILES) - -list(APPEND VAR_COMPATIBILITY_HOST_METADATA_FILES "${HOST}.meta") - -# Run ccpp_capgen -ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} - HOSTFILES ${VAR_COMPATIBILITY_HOST_METADATA_FILES} - SCHEMEFILES ${SCHEME_META_FILES} - SUITES ${SUITE_FILES} - HOST_NAME ${HOST} - OUTPUT_ROOT "${CCPP_CAP_FILES}") - -# Retrieve the list of Fortran files required for test host from datatable.xml and set to CCPP_CAPS_LIST -ccpp_datafile(DATATABLE "${CCPP_CAP_FILES}/datatable.xml" - REPORT_NAME "--ccpp-files") - -# Create test host library -add_library(VAR_COMPATIBILITY_TESTLIB OBJECT ${SCHEME_FORTRAN_FILES} - ${VAR_COMPATIBILITY_HOST_FORTRAN_FILES} - ${CCPP_CAPS_LIST}) - -# Setup test executable with needed dependencies -add_executable(var_compatibility_host_integration test_var_compatibility_integration.F90 ${HOST}.F90) -target_link_libraries(var_compatibility_host_integration PRIVATE VAR_COMPATIBILITY_TESTLIB test_utils) -target_include_directories(var_compatibility_host_integration PRIVATE "$") - -# Add executable to be called with ctest -add_test(NAME ctest_var_compatibility_host_integration COMMAND var_compatibility_host_integration) diff --git a/test/var_compatibility_test/test_host.meta b/test/var_compatibility_test/test_host.meta deleted file mode 100644 index da71b182..00000000 --- a/test/var_compatibility_test/test_host.meta +++ /dev/null @@ -1,38 +0,0 @@ -[ccpp-table-properties] - name = suite_info - type = ddt -[ccpp-arg-table] - name = suite_info - type = ddt - -[ccpp-table-properties] - name = test_host - type = host -[ccpp-arg-table] - name = test_host - type = host -[ col_start ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = () - protected = True -[ col_end ] - standard_name = horizontal_loop_end - type = integer - units = count - dimensions = () - protected = True -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = None - dimensions = () - type = character - kind = len=512 -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer diff --git a/test_prebuild/run_all_tests.sh b/test_prebuild/run_all_tests.sh deleted file mode 100755 index 08e5910a..00000000 --- a/test_prebuild/run_all_tests.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e - -echo "" && echo "Running unit_tests " && cd unit_tests && ./run_tests.sh && cd .. -echo "" && echo "Running test_opt_arg " && cd test_opt_arg && ./run_test.sh && cd .. -# No longer possible because of https://github.com/NCAR/ccpp-framework/pull/659 -#echo "" && echo "Running test_blocked_data" && cd test_blocked_data && ./run_test.sh && cd .. -echo "" && echo "Running test_chunked_data" && cd test_chunked_data && ./run_test.sh && cd .. -echo "" && echo "Running test_unit_conv" && cd test_unit_conv && ./run_test.sh && cd .. - -echo "" && echo "Running test_track_variables" && pytest test_track_variables.py diff --git a/test_prebuild/test_blocked_data/CMakeLists.txt b/test_prebuild/test_blocked_data/CMakeLists.txt deleted file mode 100644 index edce19a3..00000000 --- a/test_prebuild/test_blocked_data/CMakeLists.txt +++ /dev/null @@ -1,98 +0,0 @@ -#------------------------------------------------------------------------------ -cmake_minimum_required(VERSION 3.10) - -project(ccpp_blocked_data - VERSION 1.0.0 - LANGUAGES Fortran) - -#------------------------------------------------------------------------------ -# Request a static build -option(BUILD_SHARED_LIBS "Build a shared library" OFF) - -#------------------------------------------------------------------------------ -# Set MPI flags for C/C++/Fortran with MPI F08 interface -find_package(MPI REQUIRED Fortran) -if(NOT MPI_Fortran_HAVE_F08_MODULE) - message(FATAL_ERROR "MPI implementation does not support the Fortran 2008 mpi_f08 interface") -endif() - -#------------------------------------------------------------------------------ -# Set the sources: physics type definitions -set(TYPEDEFS $ENV{CCPP_TYPEDEFS}) -if(TYPEDEFS) - message(STATUS "Got CCPP TYPEDEFS from environment variable: ${TYPEDEFS}") -else(TYPEDEFS) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_TYPEDEFS.cmake) - message(STATUS "Got CCPP TYPEDEFS from cmakefile include file: ${TYPEDEFS}") -endif(TYPEDEFS) - -# Generate list of Fortran modules from the CCPP type -# definitions that need need to be installed -foreach(typedef_module ${TYPEDEFS}) - list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${typedef_module}) -endforeach() - -#------------------------------------------------------------------------------ -# Set the sources: physics schemes -set(SCHEMES $ENV{CCPP_SCHEMES}) -if(SCHEMES) - message(STATUS "Got CCPP SCHEMES from environment variable: ${SCHEMES}") -else(SCHEMES) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_SCHEMES.cmake) - message(STATUS "Got CCPP SCHEMES from cmakefile include file: ${SCHEMES}") -endif(SCHEMES) - -# Set the sources: physics scheme caps -set(CAPS $ENV{CCPP_CAPS}) -if(CAPS) - message(STATUS "Got CCPP CAPS from environment variable: ${CAPS}") -else(CAPS) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_CAPS.cmake) - message(STATUS "Got CCPP CAPS from cmakefile include file: ${CAPS}") -endif(CAPS) - -# Set the sources: physics scheme caps -set(API $ENV{CCPP_API}) -if(API) - message(STATUS "Got CCPP API from environment variable: ${API}") -else(API) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_API.cmake) - message(STATUS "Got CCPP API from cmakefile include file: ${API}") -endif(API) - -set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O0 -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fbounds-check -ggdb -fbacktrace -ffree-line-length-none") - -#------------------------------------------------------------------------------ -add_library(ccpp_blocked_data STATIC ${SCHEMES} ${CAPS} ${API}) -target_link_libraries(ccpp_blocked_data PRIVATE MPI::MPI_Fortran) -# Generate list of Fortran modules from defined sources -foreach(source_f90 ${CAPS} ${API}) - get_filename_component(tmp_source_f90 ${source_f90} NAME) - string(REGEX REPLACE ".F90" ".mod" tmp_module_f90 ${tmp_source_f90}) - string(TOLOWER ${tmp_module_f90} module_f90) - list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${module_f90}) -endforeach() - -set_target_properties(ccpp_blocked_data PROPERTIES VERSION ${PROJECT_VERSION} - SOVERSION ${PROJECT_VERSION_MAJOR}) - -add_executable(test_blocked_data.x main.F90) -add_dependencies(test_blocked_data.x ccpp_blocked_data) -target_link_libraries(test_blocked_data.x ccpp_blocked_data) -set_target_properties(test_blocked_data.x PROPERTIES LINKER_LANGUAGE Fortran) - -# Define where to install the library -install(TARGETS ccpp_blocked_data - EXPORT ccpp_blocked_data-targets - ARCHIVE DESTINATION lib - LIBRARY DESTINATION lib - RUNTIME DESTINATION lib -) -# Export our configuration -install(EXPORT ccpp_blocked_data-targets - FILE ccpp_blocked_data-config.cmake - DESTINATION lib/cmake -) -# Define where to install the C headers and Fortran modules -#install(FILES ${HEADERS_C} DESTINATION include) -install(FILES ${MODULES_F90} DESTINATION include) diff --git a/test_prebuild/test_blocked_data/README.md b/test_prebuild/test_blocked_data/README.md deleted file mode 100644 index 8802e812..00000000 --- a/test_prebuild/test_blocked_data/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# How to build the blocked data test - -1. Set compiler environment as appropriate for your system -2. Run the following commands: -``` -cd test_prebuild/test_blocked_data/ -rm -fr build -mkdir build -../../scripts/ccpp_prebuild.py --config=ccpp_prebuild_config.py --builddir=build -cd build -cmake .. 2>&1 | tee log.cmake -make 2>&1 | tee log.make -./test_blocked_data.x -``` diff --git a/test_prebuild/test_blocked_data/blocked_data_scheme.F90 b/test_prebuild/test_blocked_data/blocked_data_scheme.F90 deleted file mode 100644 index 77e1e687..00000000 --- a/test_prebuild/test_blocked_data/blocked_data_scheme.F90 +++ /dev/null @@ -1,126 +0,0 @@ -!>\file blocked_data_scheme.F90 -!! This file contains a blocked_data_scheme CCPP scheme that does nothing -!! except requesting the minimum, mandatory variables. - -module blocked_data_scheme - - use, intrinsic :: iso_fortran_env, only: error_unit - implicit none - - private - public :: blocked_data_scheme_init, & - blocked_data_scheme_timestep_init, & - blocked_data_scheme_run, & - blocked_data_scheme_timestep_finalize, & - blocked_data_scheme_finalize - - ! This is for unit testing only - integer, parameter, dimension(4) :: data_array_sizes = (/6, 6, 6, 3/) - -contains - - !! \section arg_table_blocked_data_scheme_init Argument Table - !! \htmlinclude blocked_data_scheme_init.html - !! - subroutine blocked_data_scheme_init(data_array, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(in) :: data_array(:) - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Check size of data array - write(error_unit, '(a,i3)') 'In blocked_data_scheme_init: checking size of data array to be', sum(data_array_sizes) - if (size(data_array)/=sum(data_array_sizes)) then - write(errmsg, '(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(& - data_array) - errflg = 1 - return - end if - end subroutine blocked_data_scheme_init - - !! \section arg_table_blocked_data_scheme_timestep_init Argument Table - !! \htmlinclude blocked_data_scheme_timestep_init.html - !! - subroutine blocked_data_scheme_timestep_init(data_array, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(in) :: data_array(:) - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Check size of data array - write(error_unit, '(a,i3)') 'In blocked_data_scheme_timestep_init: checking size of data array to be', sum(& - data_array_sizes) - if (size(data_array)/=sum(data_array_sizes)) then - write(errmsg, '(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), " but got ", size(& - data_array) - errflg = 1 - return - end if - end subroutine blocked_data_scheme_timestep_init - - !! \section arg_table_blocked_data_scheme_run Argument Table - !! \htmlinclude blocked_data_scheme_run.html - !! - subroutine blocked_data_scheme_run(nb, data_array, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(in) :: nb - integer, intent(in) :: data_array(:) - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Check size of data array - write(error_unit, '(2(a,i3))') 'In blocked_data_scheme_run: checking size of data array for block', nb, & - ' to be', data_array_sizes(nb) - if (size(data_array)/=data_array_sizes(nb)) then - write(errmsg, '(a,i4)') "Error in blocked_data_scheme_run, expected size(data_array)==6, got ", size(data_array) - errflg = 1 - return - end if - end subroutine blocked_data_scheme_run - - !! \section arg_table_blocked_data_scheme_timestep_finalize Argument Table - !! \htmlinclude blocked_data_scheme_timestep_finalize.html - !! - subroutine blocked_data_scheme_timestep_finalize(data_array, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(in) :: data_array(:) - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Check size of data array - write(error_unit, '(a,i3)') 'In blocked_data_scheme_timestep_finalize: checking size of data array to be', sum(& - data_array_sizes) - if (size(data_array)/=sum(data_array_sizes)) then - write(errmsg, '(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(& - data_array) - errflg = 1 - return - end if - end subroutine blocked_data_scheme_timestep_finalize - - !! \section arg_table_blocked_data_scheme_finalize Argument Table - !! \htmlinclude blocked_data_scheme_finalize.html - !! - subroutine blocked_data_scheme_finalize(data_array, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(in) :: data_array(:) - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Check size of data array - write(error_unit, '(a,i3)') 'In blocked_data_scheme_finalize: checking size of data array to be', sum(& - data_array_sizes) - if (size(data_array)/=sum(data_array_sizes)) then - write(errmsg, '(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(& - data_array) - errflg = 1 - return - end if - end subroutine blocked_data_scheme_finalize - -end module blocked_data_scheme diff --git a/test_prebuild/test_blocked_data/blocked_data_suite.xml b/test_prebuild/test_blocked_data/blocked_data_suite.xml deleted file mode 100644 index cf4fe9a4..00000000 --- a/test_prebuild/test_blocked_data/blocked_data_suite.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - blocked_data_scheme - - - diff --git a/test_prebuild/test_blocked_data/ccpp_prebuild_config.py b/test_prebuild/test_blocked_data/ccpp_prebuild_config.py deleted file mode 100644 index 700d9f76..00000000 --- a/test_prebuild/test_blocked_data/ccpp_prebuild_config.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -# CCPP prebuild config for GFDL Finite-Volume Cubed-Sphere Model (FV3) - -############################################################################### -# Definitions # -############################################################################### - -HOST_MODEL_IDENTIFIER = "FV3" - -# Add all files with metadata tables on the host model side and in CCPP, -# relative to basedir = top-level directory of host model. This includes -# kind and type definitions used in CCPP physics. Also add any internal -# dependencies of these files to the list. -VARIABLE_DEFINITION_FILES = [ - # actual variable definition files - '../../src/ccpp_types.F90', - 'data.F90', - ] - -TYPEDEFS_NEW_METADATA = { - 'ccpp_types' : { - 'ccpp_t' : 'cdata', - 'ccpp_types' : '', - }, - 'data' : { - 'blocked_data_type' : 'blocked_data_instance(cdata%blk_no)', - 'data' : '', - }, - } - -# Add all physics scheme files relative to basedir -SCHEME_FILES = [ - 'blocked_data_scheme.F90', - ] - -# Default build dir, relative to current working directory, -# if not specified as command-line argument -DEFAULT_BUILD_DIR = 'build' - -# Auto-generated makefile/cmakefile snippets that contain all type definitions -TYPEDEFS_MAKEFILE = '{build_dir}/CCPP_TYPEDEFS.mk' -TYPEDEFS_CMAKEFILE = '{build_dir}/CCPP_TYPEDEFS.cmake' -TYPEDEFS_SOURCEFILE = '{build_dir}/CCPP_TYPEDEFS.sh' - -# Auto-generated makefile/cmakefile snippets that contain all schemes -SCHEMES_MAKEFILE = '{build_dir}/CCPP_SCHEMES.mk' -SCHEMES_CMAKEFILE = '{build_dir}/CCPP_SCHEMES.cmake' -SCHEMES_SOURCEFILE = '{build_dir}/CCPP_SCHEMES.sh' - -# Auto-generated makefile/cmakefile snippets that contain all caps -CAPS_MAKEFILE = '{build_dir}/CCPP_CAPS.mk' -CAPS_CMAKEFILE = '{build_dir}/CCPP_CAPS.cmake' -CAPS_SOURCEFILE = '{build_dir}/CCPP_CAPS.sh' - -# Directory where to put all auto-generated physics caps -CAPS_DIR = '{build_dir}' - -# Directory where the suite definition files are stored -SUITES_DIR = '.' - -# Optional arguments - only required for schemes that use -# optional arguments. ccpp_prebuild.py will throw an exception -# if it encounters a scheme subroutine with optional arguments -# if no entry is made here. Possible values are: 'all', 'none', -# or a list of standard_names: [ 'var1', 'var3' ]. -OPTIONAL_ARGUMENTS = {} - -# Directory where to write static API to -STATIC_API_DIR = '{build_dir}' -STATIC_API_CMAKEFILE = '{build_dir}/CCPP_API.cmake' -STATIC_API_SOURCEFILE = '{build_dir}/CCPP_API.sh' - -# Directory for writing HTML pages generated from metadata files -METADATA_HTML_OUTPUT_DIR = '{build_dir}' - -# HTML document containing the model-defined CCPP variables -HTML_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_BLOCKED_DATA.html' - -# LaTeX document containing the provided vs requested CCPP variables -LATEX_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_BLOCKED_DATA.tex' diff --git a/test_prebuild/test_blocked_data/data.F90 b/test_prebuild/test_blocked_data/data.F90 deleted file mode 100644 index 0d399f27..00000000 --- a/test_prebuild/test_blocked_data/data.F90 +++ /dev/null @@ -1,41 +0,0 @@ -module data - - !! \section arg_table_data Argument Table - !! \htmlinclude data.html - !! - use ccpp_types, only: ccpp_t - - implicit none - - private - - public nblks, blksz, ncols - public ccpp_data_domain, ccpp_data_blocks, blocked_data_type, blocked_data_instance - - integer, parameter :: nblks = 4 - type(ccpp_t), target :: ccpp_data_domain - type(ccpp_t), dimension(nblks), target :: ccpp_data_blocks - - integer, parameter, dimension(nblks) :: blksz = (/6, 6, 6, 3/) - integer, parameter :: ncols = sum(blksz) - - !! \section arg_table_blocked_data_type - !! \htmlinclude blocked_data_type.html - !! - type blocked_data_type - integer, dimension(:), allocatable :: array_data - contains - procedure :: create => blocked_data_create - end type blocked_data_type - - type(blocked_data_type), dimension(nblks) :: blocked_data_instance - -contains - - subroutine blocked_data_create(blocked_data_instance, ncol) - class(blocked_data_type), intent(inout) :: blocked_data_instance - integer, intent(in) :: ncol - allocate(blocked_data_instance%array_data(ncol)) - end subroutine blocked_data_create - -end module data diff --git a/test_prebuild/test_blocked_data/data.meta b/test_prebuild/test_blocked_data/data.meta deleted file mode 100644 index c5fa2842..00000000 --- a/test_prebuild/test_blocked_data/data.meta +++ /dev/null @@ -1,69 +0,0 @@ -[ccpp-table-properties] - name = blocked_data_type - type = ddt - dependencies = -[ccpp-arg-table] - name = blocked_data_type - type = ddt -[array_data] - standard_name = blocked_data_array - long_name = blocked data array - units = 1 - dimensions = (horizontal_loop_extent) - type = integer - -[ccpp-table-properties] - name = data - type = module - dependencies = -[ccpp-arg-table] - name = data - type = module -[cdata] - standard_name = ccpp_t_instance - long_name = instance of derived data type ccpp_t - units = DDT - dimensions = () - type = ccpp_t -[nblks] - standard_name = ccpp_block_count - long_name = for explicit data blocking: number of blocks - units = count - dimensions = () - type = integer -[blksz] - standard_name = ccpp_block_sizes - long_name = for explicit data blocking: block sizes of all blocks - units = count - dimensions = (ccpp_block_count) - type = integer -[blksz(ccpp_block_number)] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent - units = count - dimensions = () - type = integer -[ncols] - standard_name = horizontal_dimension - long_name = horizontal dimension - units = count - dimensions = () - type = integer -[blocked_data_type] - standard_name = blocked_data_type - long_name = definition of type blocked_data_type - units = DDT - dimensions = () - type = blocked_data_type -[blocked_data_instance(ccpp_block_number)] - standard_name = blocked_data_type_instance - long_name = instance of derived data type blocked_data_type - units = DDT - dimensions = () - type = blocked_data_type -[blocked_data_instance] - standard_name = blocked_data_type_instance_all_blocks - long_name = instance of derived data type blocked_data_type - units = DDT - dimensions = (ccpp_block_count) - type = blocked_data_type diff --git a/test_prebuild/test_blocked_data/main.F90 b/test_prebuild/test_blocked_data/main.F90 deleted file mode 100644 index a6d86a35..00000000 --- a/test_prebuild/test_blocked_data/main.F90 +++ /dev/null @@ -1,117 +0,0 @@ -program test_blocked_data - - use, intrinsic :: iso_fortran_env, only: error_unit - - use ccpp_types, only: ccpp_t - use data, only: nblks, & - blksz, & - ncols - use data, only: ccpp_data_domain, & - ccpp_data_blocks, & - blocked_data_type, & - blocked_data_instance - - use ccpp_static_api, only: ccpp_physics_init, & - ccpp_physics_timestep_init, & - ccpp_physics_run, & - ccpp_physics_timestep_finalize, & - ccpp_physics_finalize - - implicit none - - character(len=*), parameter :: ccpp_suite = 'blocked_data_suite' - integer :: ib, ierr - type(ccpp_t), pointer :: cdata => null() - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - ! For physics running over the entire domain, - ! ccpp_thread_number and ccpp_chunk_number are - ! set to 1, indicating that arrays are to be sent - ! following their dimension specification in the - ! metadata (must match horizontal_dimension). - ccpp_data_domain%blk_no = 1 - ccpp_data_domain%thrd_no = 1 - ccpp_data_domain%thrd_cnt = 1 - - ! Loop over all blocks and threads for ccpp_data_blocks - do ib = 1, nblks - ! Assign the correct block numbers, only one thread - ccpp_data_blocks(ib)%blk_no = ib - ccpp_data_blocks(ib)%thrd_no = 1 - ccpp_data_blocks(ib)%thrd_cnt = 1 - end do - - do ib = 1, size(blocked_data_instance) - allocate(blocked_data_instance(ib)%array_data(blksz(ib))) - write(error_unit, '(2(a,i3))') "Allocated array_data for block", ib, " to size", size(blocked_data_instance(ib)%& - array_data) - end do - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - cdata => ccpp_data_domain - call ccpp_physics_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics timestep init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - cdata => ccpp_data_domain - call ccpp_physics_timestep_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics run step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - do ib = 1, nblks - cdata => ccpp_data_blocks(ib) - call ccpp_physics_run(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a,i3,a)') "An error occurred in ccpp_physics_run for block", ib, ":" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - end do - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics timestep finalize step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - cdata => ccpp_data_domain - call ccpp_physics_timestep_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_finalize:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics finalize step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - cdata => ccpp_data_domain - call ccpp_physics_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_finalize:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - -contains - -end program test_blocked_data diff --git a/test_prebuild/test_blocked_data/run_test.sh b/test_prebuild/test_blocked_data/run_test.sh deleted file mode 100755 index ee67d183..00000000 --- a/test_prebuild/test_blocked_data/run_test.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -rm -fr build -mkdir build -../../scripts/ccpp_prebuild.py --debug --config=ccpp_prebuild_config.py --builddir=build -cd build -cmake .. 2>&1 | tee log.cmake -make 2>&1 | tee log.make -./test_blocked_data.x -cd .. -rm -fr build diff --git a/test_prebuild/test_chunked_data/CMakeLists.txt b/test_prebuild/test_chunked_data/CMakeLists.txt deleted file mode 100644 index e2e7cf93..00000000 --- a/test_prebuild/test_chunked_data/CMakeLists.txt +++ /dev/null @@ -1,98 +0,0 @@ -#------------------------------------------------------------------------------ -cmake_minimum_required(VERSION 3.10) - -project(ccpp_chunked_data - VERSION 1.0.0 - LANGUAGES Fortran) - -#------------------------------------------------------------------------------ -# Request a static build -option(BUILD_SHARED_LIBS "Build a shared library" OFF) - -#------------------------------------------------------------------------------ -# Set MPI flags for C/C++/Fortran with MPI F08 interface -find_package(MPI REQUIRED Fortran) -if(NOT MPI_Fortran_HAVE_F08_MODULE) - message(FATAL_ERROR "MPI implementation does not support the Fortran 2008 mpi_f08 interface") -endif() - -#------------------------------------------------------------------------------ -# Set the sources: physics type definitions -set(TYPEDEFS $ENV{CCPP_TYPEDEFS}) -if(TYPEDEFS) - message(STATUS "Got CCPP TYPEDEFS from environment variable: ${TYPEDEFS}") -else(TYPEDEFS) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_TYPEDEFS.cmake) - message(STATUS "Got CCPP TYPEDEFS from cmakefile include file: ${TYPEDEFS}") -endif(TYPEDEFS) - -# Generate list of Fortran modules from the CCPP type -# definitions that need need to be installed -foreach(typedef_module ${TYPEDEFS}) - list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${typedef_module}) -endforeach() - -#------------------------------------------------------------------------------ -# Set the sources: physics schemes -set(SCHEMES $ENV{CCPP_SCHEMES}) -if(SCHEMES) - message(STATUS "Got CCPP SCHEMES from environment variable: ${SCHEMES}") -else(SCHEMES) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_SCHEMES.cmake) - message(STATUS "Got CCPP SCHEMES from cmakefile include file: ${SCHEMES}") -endif(SCHEMES) - -# Set the sources: physics scheme caps -set(CAPS $ENV{CCPP_CAPS}) -if(CAPS) - message(STATUS "Got CCPP CAPS from environment variable: ${CAPS}") -else(CAPS) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_CAPS.cmake) - message(STATUS "Got CCPP CAPS from cmakefile include file: ${CAPS}") -endif(CAPS) - -# Set the sources: physics scheme caps -set(API $ENV{CCPP_API}) -if(API) - message(STATUS "Got CCPP API from environment variable: ${API}") -else(API) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_API.cmake) - message(STATUS "Got CCPP API from cmakefile include file: ${API}") -endif(API) - -set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O0 -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fbounds-check -ggdb -fbacktrace -ffree-line-length-none") - -#------------------------------------------------------------------------------ -add_library(ccpp_chunked_data STATIC ${SCHEMES} ${CAPS} ${API}) -target_link_libraries(ccpp_chunked_data PRIVATE MPI::MPI_Fortran) -# Generate list of Fortran modules from defined sources -foreach(source_f90 ${CAPS} ${API}) - get_filename_component(tmp_source_f90 ${source_f90} NAME) - string(REGEX REPLACE ".F90" ".mod" tmp_module_f90 ${tmp_source_f90}) - string(TOLOWER ${tmp_module_f90} module_f90) - list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${module_f90}) -endforeach() - -set_target_properties(ccpp_chunked_data PROPERTIES VERSION ${PROJECT_VERSION} - SOVERSION ${PROJECT_VERSION_MAJOR}) - -add_executable(test_chunked_data.x main.F90) -add_dependencies(test_chunked_data.x ccpp_chunked_data) -target_link_libraries(test_chunked_data.x ccpp_chunked_data) -set_target_properties(test_chunked_data.x PROPERTIES LINKER_LANGUAGE Fortran) - -# Define where to install the library -install(TARGETS ccpp_chunked_data - EXPORT ccpp_chunked_data-targets - ARCHIVE DESTINATION lib - LIBRARY DESTINATION lib - RUNTIME DESTINATION lib -) -# Export our configuration -install(EXPORT ccpp_chunked_data-targets - FILE ccpp_chunked_data-config.cmake - DESTINATION lib/cmake -) -# Define where to install the C headers and Fortran modules -#install(FILES ${HEADERS_C} DESTINATION include) -install(FILES ${MODULES_F90} DESTINATION include) diff --git a/test_prebuild/test_chunked_data/ccpp_prebuild_config.py b/test_prebuild/test_chunked_data/ccpp_prebuild_config.py deleted file mode 100755 index 4e32d37d..00000000 --- a/test_prebuild/test_chunked_data/ccpp_prebuild_config.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -# CCPP prebuild config for GFDL Finite-Volume Cubed-Sphere Model (FV3) - -############################################################################### -# Definitions # -############################################################################### - -HOST_MODEL_IDENTIFIER = "FV3" - -# Add all files with metadata tables on the host model side and in CCPP, -# relative to basedir = top-level directory of host model. This includes -# kind and type definitions used in CCPP physics. Also add any internal -# dependencies of these files to the list. -VARIABLE_DEFINITION_FILES = [ - # actual variable definition files - '../../src/ccpp_types.F90', - 'data.F90', - ] - -TYPEDEFS_NEW_METADATA = { - 'ccpp_types' : { - 'ccpp_t' : 'cdata', - 'ccpp_types' : '', - }, - 'data' : { - 'chunked_data_type' : 'chunked_data_instance', - 'data' : '', - }, - } - -# Add all physics scheme files relative to basedir -SCHEME_FILES = [ - 'chunked_data_scheme.F90', - ] - -# Default build dir, relative to current working directory, -# if not specified as command-line argument -DEFAULT_BUILD_DIR = 'build' - -# Auto-generated makefile/cmakefile snippets that contain all type definitions -TYPEDEFS_MAKEFILE = '{build_dir}/CCPP_TYPEDEFS.mk' -TYPEDEFS_CMAKEFILE = '{build_dir}/CCPP_TYPEDEFS.cmake' -TYPEDEFS_SOURCEFILE = '{build_dir}/CCPP_TYPEDEFS.sh' - -# Auto-generated makefile/cmakefile snippets that contain all schemes -SCHEMES_MAKEFILE = '{build_dir}/CCPP_SCHEMES.mk' -SCHEMES_CMAKEFILE = '{build_dir}/CCPP_SCHEMES.cmake' -SCHEMES_SOURCEFILE = '{build_dir}/CCPP_SCHEMES.sh' - -# Auto-generated makefile/cmakefile snippets that contain all caps -CAPS_MAKEFILE = '{build_dir}/CCPP_CAPS.mk' -CAPS_CMAKEFILE = '{build_dir}/CCPP_CAPS.cmake' -CAPS_SOURCEFILE = '{build_dir}/CCPP_CAPS.sh' - -# Directory where to put all auto-generated physics caps -CAPS_DIR = '{build_dir}' - -# Directory where the suite definition files are stored -SUITES_DIR = '.' - -# Optional arguments - only required for schemes that use -# optional arguments. ccpp_prebuild.py will throw an exception -# if it encounters a scheme subroutine with optional arguments -# if no entry is made here. Possible values are: 'all', 'none', -# or a list of standard_names: [ 'var1', 'var3' ]. -OPTIONAL_ARGUMENTS = {} - -# Directory where to write static API to -STATIC_API_DIR = '{build_dir}' -STATIC_API_CMAKEFILE = '{build_dir}/CCPP_API.cmake' -STATIC_API_SOURCEFILE = '{build_dir}/CCPP_API.sh' - -# Directory for writing HTML pages generated from metadata files -METADATA_HTML_OUTPUT_DIR = '{build_dir}' - -# HTML document containing the model-defined CCPP variables -HTML_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_CHUNKED_DATA.html' - -# LaTeX document containing the provided vs requested CCPP variables -LATEX_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_CHUNKED_DATA.tex' diff --git a/test_prebuild/test_chunked_data/data.meta b/test_prebuild/test_chunked_data/data.meta deleted file mode 100644 index c14217df..00000000 --- a/test_prebuild/test_chunked_data/data.meta +++ /dev/null @@ -1,76 +0,0 @@ -[ccpp-table-properties] - name = chunked_data_type - type = ddt - dependencies = -[ccpp-arg-table] - name = chunked_data_type - type = ddt -[array_data] - standard_name = chunked_data_array - long_name = chunked data array - units = 1 - dimensions = (ccpp_constant_one:horizontal_dimension) - type = integer -# Todo: define an additional array running from -1 to ncols-2 - -[ccpp-table-properties] - name = data - type = module - dependencies = -[ccpp-arg-table] - name = data - type = module -[cdata] - standard_name = ccpp_t_instance - long_name = instance of derived data type ccpp_t - units = DDT - dimensions = () - type = ccpp_t -[ncols] - standard_name = horizontal_dimension - long_name = horizontal dimension - units = count - dimensions = () - type = integer -[nchunks] - standard_name = ccpp_chunk_extent - long_name = number of chunks of array data used in run phase - units = count - dimensions = () - type = integer -[chunk_begin] - standard_name = horizontal_loop_begin_all_chunks - long_name = first index for horizontal loop extent in run phase - units = index - dimensions = (ccpp_chunk_extent) - type = integer -[chunk_begin(ccpp_chunk_number)] - standard_name = horizontal_loop_begin - long_name = first index for horizontal loop extent in run phase - units = index - dimensions = () - type = integer -[chunk_end] - standard_name = horizontal_loop_end_all_chunks - long_name = last index for horizontal loop extent in run phase - units = index - dimensions = (ccpp_chunk_extent) - type = integer -[chunk_end(ccpp_chunk_number)] - standard_name = horizontal_loop_end - long_name = last index for horizontal loop extent in run phase - units = index - dimensions = () - type = integer -[chunked_data_type] - standard_name = chunked_data_type - long_name = definition of type chunked_data_type - units = DDT - dimensions = () - type = chunked_data_type -[chunked_data_instance] - standard_name = chunked_data_type_instance - long_name = instance of derived data type chunked_data_type - units = DDT - dimensions = () - type = chunked_data_type diff --git a/test_prebuild/test_chunked_data/main.F90 b/test_prebuild/test_chunked_data/main.F90 deleted file mode 100644 index 739ebf8b..00000000 --- a/test_prebuild/test_chunked_data/main.F90 +++ /dev/null @@ -1,114 +0,0 @@ -program test_chunked_data - - use, intrinsic :: iso_fortran_env, only: error_unit - - use ccpp_types, only: ccpp_t - use data, only: nchunks, & - chunksize, & - ncols - use data, only: ccpp_data_domain, & - ccpp_data_chunks, & - chunked_data_type, & - chunked_data_instance - - use ccpp_static_api, only: ccpp_physics_init, & - ccpp_physics_timestep_init, & - ccpp_physics_run, & - ccpp_physics_timestep_finalize, & - ccpp_physics_finalize - - implicit none - - character(len=*), parameter :: ccpp_suite = 'chunked_data_suite' - integer :: ic, ierr - type(ccpp_t), pointer :: cdata => null() - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - ! For physics running over the entire domain, - ! ccpp_thread_number and ccpp_chunk_number are - ! set to 1, indicating that arrays are to be sent - ! following their dimension specification in the - ! metadata (must match horizontal_dimension). - ccpp_data_domain%thrd_no = 1 - ccpp_data_domain%chunk_no = 1 - ccpp_data_domain%thrd_cnt = 1 - - ! Loop over all chunks and threads for ccpp_data_chunks - do ic = 1, nchunks - ! Assign the correct chunk numbers, only one thread - ccpp_data_chunks(ic)%chunk_no = ic - ccpp_data_chunks(ic)%thrd_no = 1 - ccpp_data_chunks(ic)%thrd_cnt = 1 - end do - - call chunked_data_instance%create(ncols) - write(error_unit, '(2(a,i3))') "Chunked_data_instance%array_data to size", size(chunked_data_instance%array_data) - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - cdata => ccpp_data_domain - call ccpp_physics_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics timestep init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - cdata => ccpp_data_domain - call ccpp_physics_timestep_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics run step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - do ic = 1, nchunks - cdata => ccpp_data_chunks(ic) - call ccpp_physics_run(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a,i3,a)') "An error occurred in ccpp_physics_run for chunk", ic, ":" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - end do - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics timestep finalize step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - cdata => ccpp_data_domain - call ccpp_physics_timestep_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_finalize:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics finalize step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - cdata => ccpp_data_domain - call ccpp_physics_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_finalize:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - -contains - -end program test_chunked_data diff --git a/test_prebuild/test_chunked_data/run_test.sh b/test_prebuild/test_chunked_data/run_test.sh deleted file mode 100755 index 9ba2a36e..00000000 --- a/test_prebuild/test_chunked_data/run_test.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -e - -rm -fr build -mkdir build -../../scripts/ccpp_prebuild.py --debug --config=ccpp_prebuild_config.py --builddir=build -cd build -cmake .. 2>&1 | tee log.cmake -make 2>&1 | tee log.make -./test_chunked_data.x -cd .. -rm -fr build diff --git a/test_prebuild/test_opt_arg/CMakeLists.txt b/test_prebuild/test_opt_arg/CMakeLists.txt deleted file mode 100644 index 58e6e6b5..00000000 --- a/test_prebuild/test_opt_arg/CMakeLists.txt +++ /dev/null @@ -1,98 +0,0 @@ -#------------------------------------------------------------------------------ -cmake_minimum_required(VERSION 3.10) - -project(ccpp_opt_arg - VERSION 1.0.0 - LANGUAGES Fortran) - -#------------------------------------------------------------------------------ -# Request a static build -option(BUILD_SHARED_LIBS "Build a shared library" OFF) - -#------------------------------------------------------------------------------ -# Set MPI flags for C/C++/Fortran with MPI F08 interface -find_package(MPI REQUIRED Fortran) -if(NOT MPI_Fortran_HAVE_F08_MODULE) - message(FATAL_ERROR "MPI implementation does not support the Fortran 2008 mpi_f08 interface") -endif() - -#------------------------------------------------------------------------------ -# Set the sources: physics type definitions -set(TYPEDEFS $ENV{CCPP_TYPEDEFS}) -if(TYPEDEFS) - message(STATUS "Got CCPP TYPEDEFS from environment variable: ${TYPEDEFS}") -else(TYPEDEFS) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_TYPEDEFS.cmake) - message(STATUS "Got CCPP TYPEDEFS from cmakefile include file: ${TYPEDEFS}") -endif(TYPEDEFS) - -# Generate list of Fortran modules from the CCPP type -# definitions that need need to be installed -foreach(typedef_module ${TYPEDEFS}) - list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${typedef_module}) -endforeach() - -#------------------------------------------------------------------------------ -# Set the sources: physics schemes -set(SCHEMES $ENV{CCPP_SCHEMES}) -if(SCHEMES) - message(STATUS "Got CCPP SCHEMES from environment variable: ${SCHEMES}") -else(SCHEMES) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_SCHEMES.cmake) - message(STATUS "Got CCPP SCHEMES from cmakefile include file: ${SCHEMES}") -endif(SCHEMES) - -# Set the sources: physics scheme caps -set(CAPS $ENV{CCPP_CAPS}) -if(CAPS) - message(STATUS "Got CCPP CAPS from environment variable: ${CAPS}") -else(CAPS) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_CAPS.cmake) - message(STATUS "Got CCPP CAPS from cmakefile include file: ${CAPS}") -endif(CAPS) - -# Set the sources: physics scheme caps -set(API $ENV{CCPP_API}) -if(API) - message(STATUS "Got CCPP API from environment variable: ${API}") -else(API) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_API.cmake) - message(STATUS "Got CCPP API from cmakefile include file: ${API}") -endif(API) - -set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O0 -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fcheck=all -ggdb -fbacktrace -ffree-line-length-none") - -#------------------------------------------------------------------------------ -add_library(ccpp_opt_arg STATIC ${SCHEMES} ${CAPS} ${API}) -target_link_libraries(ccpp_opt_arg PRIVATE MPI::MPI_Fortran) -# Generate list of Fortran modules from defined sources -foreach(source_f90 ${CAPS} ${API}) - get_filename_component(tmp_source_f90 ${source_f90} NAME) - string(REGEX REPLACE ".F90" ".mod" tmp_module_f90 ${tmp_source_f90}) - string(TOLOWER ${tmp_module_f90} module_f90) - list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${module_f90}) -endforeach() - -set_target_properties(ccpp_opt_arg PROPERTIES VERSION ${PROJECT_VERSION} - SOVERSION ${PROJECT_VERSION_MAJOR}) - -add_executable(test_opt_arg.x main.F90) -add_dependencies(test_opt_arg.x ccpp_opt_arg) -target_link_libraries(test_opt_arg.x ccpp_opt_arg) -set_target_properties(test_opt_arg.x PROPERTIES LINKER_LANGUAGE Fortran) - -# Define where to install the library -install(TARGETS ccpp_opt_arg - EXPORT ccpp_opt_arg-targets - ARCHIVE DESTINATION lib - LIBRARY DESTINATION lib - RUNTIME DESTINATION lib -) -# Export our configuration -install(EXPORT ccpp_opt_arg-targets - FILE ccpp_opt_arg-config.cmake - DESTINATION lib/cmake -) -# Define where to install the C headers and Fortran modules -#install(FILES ${HEADERS_C} DESTINATION include) -install(FILES ${MODULES_F90} DESTINATION include) diff --git a/test_prebuild/test_opt_arg/ccpp_kinds.F90 b/test_prebuild/test_opt_arg/ccpp_kinds.F90 deleted file mode 100644 index a07ded9b..00000000 --- a/test_prebuild/test_opt_arg/ccpp_kinds.F90 +++ /dev/null @@ -1,13 +0,0 @@ -module ccpp_kinds - - !! \section arg_table_ccpp_kinds - !! \htmlinclude ccpp_kinds.html - !! - - use iso_fortran_env, only: real64 - - implicit none - - integer, parameter :: kind_phys = real64 - -end module ccpp_kinds diff --git a/test_prebuild/test_opt_arg/ccpp_kinds.meta b/test_prebuild/test_opt_arg/ccpp_kinds.meta deleted file mode 100644 index 0e95702e..00000000 --- a/test_prebuild/test_opt_arg/ccpp_kinds.meta +++ /dev/null @@ -1,15 +0,0 @@ -[ccpp-table-properties] - name = ccpp_kinds - type = module - dependencies = - -######################################################################## -[ccpp-arg-table] - name = ccpp_kinds - type = module -[kind_phys] - standard_name = kind_phys - long_name = definition of kind_phys - units = none - dimensions = () - type = integer diff --git a/test_prebuild/test_opt_arg/ccpp_prebuild_config.py b/test_prebuild/test_opt_arg/ccpp_prebuild_config.py deleted file mode 100755 index bf3bc1cc..00000000 --- a/test_prebuild/test_opt_arg/ccpp_prebuild_config.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -# CCPP prebuild config for GFDL Finite-Volume Cubed-Sphere Model (FV3) - -############################################################################### -# Definitions # -############################################################################### - -HOST_MODEL_IDENTIFIER = "FV3" - -# Add all files with metadata tables on the host model side and in CCPP, -# relative to basedir = top-level directory of host model. This includes -# kind and type definitions used in CCPP physics. Also add any internal -# dependencies of these files to the list. -VARIABLE_DEFINITION_FILES = [ - # actual variable definition files - '../../src/ccpp_types.F90', - 'ccpp_kinds.F90', - 'data.F90', - ] - -TYPEDEFS_NEW_METADATA = { - 'ccpp_types' : { - 'ccpp_t' : 'cdata', - 'ccpp_types' : '', - }, - 'data' : { - 'data' : '', - }, - } - -# Add all physics scheme files relative to basedir -SCHEME_FILES = [ - 'opt_arg_scheme.F90', - ] - -# Default build dir, relative to current working directory, -# if not specified as command-line argument -DEFAULT_BUILD_DIR = 'build' - -# Auto-generated makefile/cmakefile snippets that contain all type definitions -TYPEDEFS_MAKEFILE = '{build_dir}/CCPP_TYPEDEFS.mk' -TYPEDEFS_CMAKEFILE = '{build_dir}/CCPP_TYPEDEFS.cmake' -TYPEDEFS_SOURCEFILE = '{build_dir}/CCPP_TYPEDEFS.sh' - -# Auto-generated makefile/cmakefile snippets that contain all schemes -SCHEMES_MAKEFILE = '{build_dir}/CCPP_SCHEMES.mk' -SCHEMES_CMAKEFILE = '{build_dir}/CCPP_SCHEMES.cmake' -SCHEMES_SOURCEFILE = '{build_dir}/CCPP_SCHEMES.sh' - -# Auto-generated makefile/cmakefile snippets that contain all caps -CAPS_MAKEFILE = '{build_dir}/CCPP_CAPS.mk' -CAPS_CMAKEFILE = '{build_dir}/CCPP_CAPS.cmake' -CAPS_SOURCEFILE = '{build_dir}/CCPP_CAPS.sh' - -# Directory where to put all auto-generated physics caps -CAPS_DIR = '{build_dir}' - -# Directory where the suite definition files are stored -SUITES_DIR = '.' - -# Optional arguments - only required for schemes that use -# optional arguments. ccpp_prebuild.py will throw an exception -# if it encounters a scheme subroutine with optional arguments -# if no entry is made here. Possible values are: 'all', 'none', -# or a list of standard_names: [ 'var1', 'var3' ]. -OPTIONAL_ARGUMENTS = {} - -# Directory where to write static API to -STATIC_API_DIR = '{build_dir}' -STATIC_API_CMAKEFILE = '{build_dir}/CCPP_API.cmake' -STATIC_API_SOURCEFILE = '{build_dir}/CCPP_API.sh' - -# Directory for writing HTML pages generated from metadata files -METADATA_HTML_OUTPUT_DIR = '{build_dir}' - -# HTML document containing the model-defined CCPP variables -HTML_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_opt_arg.html' - -# LaTeX document containing the provided vs requested CCPP variables -LATEX_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_opt_arg.tex' diff --git a/test_prebuild/test_opt_arg/main.F90 b/test_prebuild/test_opt_arg/main.F90 deleted file mode 100644 index 8d08619a..00000000 --- a/test_prebuild/test_opt_arg/main.F90 +++ /dev/null @@ -1,125 +0,0 @@ -program test_opt_arg - - use, intrinsic :: iso_fortran_env, only: output_unit, & - error_unit - - use ccpp_types, only: ccpp_t - use data, only: cdata, & - nx, & - flag_for_opt_arg, & - std_arg, & - opt_arg, & - opt_arg_2 - - use ccpp_static_api, only: ccpp_physics_init, & - ccpp_physics_timestep_init, & - ccpp_physics_run, & - ccpp_physics_timestep_finalize, & - ccpp_physics_finalize - - implicit none - - character(len=*), parameter :: ccpp_suite = 'opt_arg_suite' - integer :: ierr - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - cdata%blk_no = 1 - cdata%thrd_no = 1 - cdata%thrd_cnt = 1 - - std_arg = 1 - flag_for_opt_arg = .true. - allocate(opt_arg(nx)) - allocate(opt_arg_2(nx)) - - ! std_arg must all be 1, opt_arg must all be 0 - write(output_unit, '(a)') "After ccpp_init: check std_arg(:)==1, opt_arg(:)==0, opt_arg_2(:)==0" - if (.not. all(std_arg == 1)) write(error_unit, '(a,3i3)') "Error after ccpp_init: std_arg=", std_arg - if (.not. all(opt_arg == 0)) write(error_unit, '(a,3i3)') "Error after ccpp_init: opt_arg=", opt_arg - if (.not. all(opt_arg_2 == 0)) write(error_unit, '(a,3i3)') "Error after ccpp_init: opt_arg_2=", opt_arg_2 - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - call ccpp_physics_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - ! std_arg must all be 1, opt_arg must all be 0 - write(output_unit, '(a)') "After ccpp_physics_init: check std_arg(:)==1 and opt_arg(:)==0" - if (.not. all(std_arg == 1)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_init: std_arg=", std_arg - if (.not. all(opt_arg == 0)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_init: opt_arg=", opt_arg - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics timestep init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - call ccpp_physics_timestep_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - ! std_arg must all be 1, opt_arg must all be 2 - write(output_unit, '(a)') "After ccpp_physics_timestep_init: check std_arg(:)==1 and opt_arg(:)==2" - if (.not. all(std_arg == 1)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_timestep_init: std_arg=", std_arg - if (.not. all(opt_arg == 2)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_timestep_init: opt_arg=", opt_arg - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics run step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - call ccpp_physics_run(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_run:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - ! std_arg must all be 1, opt_arg must all be 3 - write(output_unit, '(a)') "After ccpp_physics_run: check std_arg(:)==1 and opt_arg(:)==3" - if (.not. all(std_arg == 1)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_run: std_arg=", std_arg - if (.not. all(opt_arg == 3)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_run: opt_arg=", opt_arg - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics timestep finalize step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - deallocate(opt_arg) - flag_for_opt_arg = .false. - - call ccpp_physics_timestep_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - ! std_arg must all be 7, opt_arg no longer allocated - write(output_unit, '(a)') "After ccpp_physics_timestep_final: check std_arg(:)==7; opt_arg not allocated" - if (.not. all(std_arg == 7)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_timestep_final: std_arg=", std_arg - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics finalize step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - call ccpp_physics_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - ! std_arg must all be 7, opt_arg no longer allocated - write(output_unit, '(a)') "After ccpp_physics_timestep_final: check std_arg(:)==7; opt_arg not allocated" - if (.not. all(std_arg == 7)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_timestep_final: std_arg=", std_arg - -end program test_opt_arg diff --git a/test_prebuild/test_opt_arg/run_test.sh b/test_prebuild/test_opt_arg/run_test.sh deleted file mode 100755 index cbe50e6d..00000000 --- a/test_prebuild/test_opt_arg/run_test.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -e - -rm -fr build -mkdir build -../../scripts/ccpp_prebuild.py --debug --config=ccpp_prebuild_config.py --builddir=build -cd build -cmake .. 2>&1 | tee log.cmake -make 2>&1 | tee log.make -./test_opt_arg.x -cd .. -rm -fr build diff --git a/test_prebuild/test_track_variables.py b/test_prebuild/test_track_variables.py deleted file mode 100755 index 6f2b1963..00000000 --- a/test_prebuild/test_track_variables.py +++ /dev/null @@ -1,130 +0,0 @@ -#! /usr/bin/env python3 -""" ------------------------------------------------------------------------ - Description: Contains unit tests for ccpp_track_variables.py script - - Assumptions: Assumes user has correct environment for running ccpp_track_variables.py script. - This script should not be run directly, but rather invoked with pytest. - - Command line arguments: none - - Usage: pytest test_track_variables.py # run the unit tests ------------------------------------------------------------------------ -""" -import sys -import os -import pytest - -TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -SCRIPTS_DIR = os.path.abspath(os.path.join(TEST_DIR, os.pardir, "scripts")) -SAMPLE_FILES_DIR = "test_track_variables" -SUITE_FILE = f'{SAMPLE_FILES_DIR}/suite_TEST_SUITE.xml' -SMALL_SUITE_FILE = f'{SAMPLE_FILES_DIR}/suite_small_suite.xml' -CONFIG_FILE = f'{SAMPLE_FILES_DIR}/ccpp_prebuild_config.py' -if not os.path.exists(SCRIPTS_DIR): - raise ImportError(f"Cannot find scripts directory {SCRIPTS_DIR}") - -sys.path.append(SCRIPTS_DIR) - -from ccpp_track_variables import track_variables - -def test_successful_match(capsys): - """Tests whether test_track_variables.py produces expected output from sample suite and - metadata files for a case with a successful match (user provided a variable that exists - within the schemes specified by the test suite)""" - expected_output = """For suite test_track_variables/suite_small_suite.xml, the following schemes (in order for each group) use the variable air_pressure: -In group group1 - scheme_1_run (intent in) - scheme_1_run (intent in)""" - track_variables(SMALL_SUITE_FILE,SAMPLE_FILES_DIR,CONFIG_FILE,'air_pressure',False) - streams = capsys.readouterr() - expected_output_list = expected_output.splitlines() - streams_err_list = streams.err.splitlines() - for (err, expected) in zip(streams_err_list, expected_output_list): - assert err.strip() == expected.strip() - -def test_successful_match_with_subcycles(capsys): - """Tests whether test_track_variables.py produces expected output from sample suite and - metadata files for a case with a successful match (user provided a variable that exists - within the schemes specified by the test suite). In this case, the test suite file - contains subcycles, so the output should reflect this.""" - - expected_output = """For suite test_track_variables/suite_TEST_SUITE.xml, the following schemes (in order for each group) use the variable surface_air_pressure: -In group group1 - scheme_3_run (intent inout) - scheme_3_timestep_finalize (intent inout) - scheme_3_timestep_finalize (intent out) - scheme_4_run (intent in) - scheme_3_run (intent inout) - scheme_3_timestep_finalize (intent inout) - scheme_3_timestep_finalize (intent out) - scheme_4_run (intent in) -In group group2 - scheme_4_run (intent in) - scheme_4_run (intent in) - scheme_4_run (intent in)""" - track_variables(SUITE_FILE,SAMPLE_FILES_DIR,CONFIG_FILE,'surface_air_pressure',False) - streams = capsys.readouterr() - expected_output_list = expected_output.splitlines() - streams_err_list = streams.err.splitlines() - for (err, expected) in zip(streams_err_list, expected_output_list): - assert err.strip() == expected.strip() - - -def test_partial_match(capsys): - """Tests whether test_track_variables.py produces expected output from sample suite and - metadata files for a case with a partial match: user provided a variable that does not - exist in the test suite, but is a substring of one or more other variables that do - exist.""" - - expected_output = """Variable surface not found in any suites for sdf test_track_variables/suite_TEST_SUITE.xml - -ERROR:ccpp_track_variables:Variable surface not found in any suites for sdf test_track_variables/suite_TEST_SUITE.xml - -Did find partial matches that may be of interest: - -In scheme_2_init found variable(s) ['surface_emissivity_data_file'] -In scheme_2_run found variable(s) ['surface_roughness_length', 'surface_ground_temperature_for_radiation', 'surface_air_temperature_for_radiation', 'surface_skin_temperature_over_ice', 'baseline_surface_longwave_emissivity', 'surface_longwave_emissivity', 'surface_albedo_components', 'surface_albedo_for_diffused_shortwave_on_radiation_timestep'] -In scheme_3_init found variable(s) ['flag_for_mellor_yamada_nakanishi_niino_surface_layer_scheme'] -In scheme_3_timestep_init found variable(s) ['flag_for_mellor_yamada_nakanishi_niino_surface_layer_scheme'] -In scheme_3_run found variable(s) ['do_compute_surface_scalar_fluxes', 'do_compute_surface_diagnostics', 'surface_air_pressure', 'reference_air_pressure_normalized_by_surface_air_pressure'] -In scheme_3_timestep_finalize found variable(s) ['surface_air_pressure'] -In scheme_4_run found variable(s) ['surface_air_pressure'] -In scheme_B_run found variable(s) ['flag_nonzero_wet_surface_fraction', 'sea_surface_temperature', 'surface_skin_temperature_after_iteration_over_water'] -""" - track_variables(SUITE_FILE,SAMPLE_FILES_DIR,CONFIG_FILE,'surface',False) - streams = capsys.readouterr() - expected_output_list = expected_output.splitlines() - streams_err_list = streams.err.splitlines() - for (err, expected) in zip(streams_err_list, expected_output_list): - assert err.strip() == expected.strip() - - -def test_no_match(capsys): - """Tests whether test_track_variables.py produces expected output from sample suite and - metadata files for a case with no match (user provided a variable that does not exist - within the schemes specified by the test suite)""" - - expected_output = """Variable abc not found in any suites for sdf test_track_variables/suite_TEST_SUITE.xml - -ERROR:ccpp_track_variables:Variable abc not found in any suites for sdf test_track_variables/suite_TEST_SUITE.xml""" - track_variables(SUITE_FILE,SAMPLE_FILES_DIR,CONFIG_FILE,'abc',False) - streams = capsys.readouterr() - expected_output_list = expected_output.splitlines() - streams_err_list = streams.err.splitlines() - for (err, expected) in zip(streams_err_list, expected_output_list): - assert err.strip() == expected.strip() - - -def test_bad_config(capsys): - """Tests whether test_track_variables.py fails gracefully when provided a config file that does - not exist.""" - with pytest.raises(Exception) as excinfo: - track_variables(SUITE_FILE,SAMPLE_FILES_DIR,f'{SAMPLE_FILES_DIR}/nofile','abc',False) - assert str(excinfo.value) == "Call to import_config failed." - - -if __name__ == "__main__": - print("This test file is designed to be run with pytest; can not be run directly") - sys.exit(1) - diff --git a/test_prebuild/test_track_variables/ccpp_prebuild_config.py b/test_prebuild/test_track_variables/ccpp_prebuild_config.py deleted file mode 100755 index 83b20fd1..00000000 --- a/test_prebuild/test_track_variables/ccpp_prebuild_config.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python - -# CCPP prebuild config for CCPP track variables tool test - - -############################################################################### -# Definitions # -############################################################################### - -HOST_MODEL_IDENTIFIER = "TEST" - -# Add all files with metadata tables on the host model side and in CCPP, -# relative to basedir = top-level directory of host model. This includes -# kind and type definitions used in CCPP physics. Also add any internal -# dependencies of these files to the list. -VARIABLE_DEFINITION_FILES = [ - # actual variable definition files - '../src/ccpp_types.F90', - ] - -# Add all physics scheme files relative to basedir -SCHEME_FILES = [ ] - -# Default build dir, relative to current working directory, -# if not specified as command-line argument -DEFAULT_BUILD_DIR = 'build' - -# Auto-generated makefile/cmakefile snippets that contain all type definitions -TYPEDEFS_MAKEFILE = '{build_dir}/CCPP_TYPEDEFS.mk' -TYPEDEFS_CMAKEFILE = '{build_dir}/CCPP_TYPEDEFS.cmake' -TYPEDEFS_SOURCEFILE = '{build_dir}/CCPP_TYPEDEFS.sh' - -# Auto-generated makefile/cmakefile snippets that contain all schemes -SCHEMES_MAKEFILE = '{build_dir}/CCPP_SCHEMES.mk' -SCHEMES_CMAKEFILE = '{build_dir}/CCPP_SCHEMES.cmake' -SCHEMES_SOURCEFILE = '{build_dir}/CCPP_SCHEMES.sh' - -# Auto-generated makefile/cmakefile snippets that contain all caps -CAPS_MAKEFILE = '{build_dir}/CCPP_CAPS.mk' -CAPS_CMAKEFILE = '{build_dir}/CCPP_CAPS.cmake' -CAPS_SOURCEFILE = '{build_dir}/CCPP_CAPS.sh' - -# Directory where to put all auto-generated physics caps -CAPS_DIR = '{build_dir}' - -# Directory where the suite definition files are stored -SUITES_DIR = '.' - -# Optional arguments - only required for schemes that use -# optional arguments. ccpp_prebuild.py will throw an exception -# if it encounters a scheme subroutine with optional arguments -# if no entry is made here. Possible values are: 'all', 'none', -# or a list of standard_names: [ 'var1', 'var3' ]. -OPTIONAL_ARGUMENTS = {} - -# Directory where to write static API to -STATIC_API_DIR = '{build_dir}' -STATIC_API_CMAKEFILE = '{build_dir}/CCPP_API.cmake' -STATIC_API_SOURCEFILE = '{build_dir}/CCPP_API.sh' - -# Directory for writing HTML pages generated from metadata files -METADATA_HTML_OUTPUT_DIR = '{build_dir}' - -# HTML document containing the model-defined CCPP variables -HTML_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_BLOCKED_DATA.html' - -# LaTeX document containing the provided vs requested CCPP variables -LATEX_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_BLOCKED_DATA.tex' - diff --git a/test_prebuild/test_track_variables/scheme_1.meta b/test_prebuild/test_track_variables/scheme_1.meta deleted file mode 100644 index 8ee0800a..00000000 --- a/test_prebuild/test_track_variables/scheme_1.meta +++ /dev/null @@ -1,25 +0,0 @@ -[ccpp-table-properties] - name = scheme_1 - type = scheme - dependencies = ../../hooks/machine.F - -######################################################################## -[ccpp-arg-table] - name = scheme_1_run - type = scheme -[p_lay] - standard_name = air_pressure - long_name = mean layer pressure - units = Pa - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = in -[p_lev] - standard_name = air_pressure_at_interface - long_name = air pressure at model layer interfaces - units = Pa - dimensions = (horizontal_loop_extent,vertical_interface_dimension) - type = real - kind = kind_phys - intent = in diff --git a/test_prebuild/test_track_variables/scheme_2.meta b/test_prebuild/test_track_variables/scheme_2.meta deleted file mode 100644 index 2a03f096..00000000 --- a/test_prebuild/test_track_variables/scheme_2.meta +++ /dev/null @@ -1,87 +0,0 @@ -[ccpp-table-properties] - name = scheme_2 - type = scheme - dependencies_path = ../../ - dependencies = hooks/machine.F - -######################################################################## -[ccpp-arg-table] - name = scheme_2_init - type = scheme -[semis_file] - standard_name = surface_emissivity_data_file - long_name = surface emissivity data file for radiation - units = none - dimensions = () - type = character - kind = len=26 - intent = in - -######################################################################## -[ccpp-arg-table] - name = scheme_2_run - type = scheme -[zorl] - standard_name = surface_roughness_length - long_name = surface roughness length - units = cm - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[tsfg] - standard_name = surface_ground_temperature_for_radiation - long_name = surface ground temperature for radiation - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[tsfa] - standard_name = surface_air_temperature_for_radiation - long_name = lowest model layer air temperature for radiation - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[tisfc] - standard_name = surface_skin_temperature_over_ice - long_name = surface_skin_temperature_over_ice - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[semisbase] - standard_name = baseline_surface_longwave_emissivity - long_name = baseline surface lw emissivity in fraction - units = frac - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[semis] - standard_name = surface_longwave_emissivity - long_name = surface lw emissivity in fraction - units = frac - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[sfcalb] - standard_name = surface_albedo_components - long_name = surface albedo IR/UV/VIS components - units = frac - dimensions = (horizontal_loop_extent,number_of_components_for_surface_albedo) - type = real - kind = kind_phys - intent = inout -[sfc_alb_dif] - standard_name = surface_albedo_for_diffused_shortwave_on_radiation_timestep - long_name = mean surface diffused sw albedo - units = frac - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout diff --git a/test_prebuild/test_track_variables/scheme_3.meta b/test_prebuild/test_track_variables/scheme_3.meta deleted file mode 100644 index f7d568dd..00000000 --- a/test_prebuild/test_track_variables/scheme_3.meta +++ /dev/null @@ -1,143 +0,0 @@ -[ccpp-table-properties] - name = scheme_3 - type = scheme - dependencies = ../../hooks/machine.F,../../hooks/physcons.F90,module_sf_mynn.F90 -######################################################################## -[ccpp-arg-table] - name = scheme_3_init - type = scheme -[do_mynnsfclay] - standard_name = flag_for_mellor_yamada_nakanishi_niino_surface_layer_scheme - long_name = flag to activate MYNN surface layer - units = flag - dimensions = () - type = logical - intent = in -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out - -######################################################################## -[ccpp-arg-table] - name = scheme_3_timestep_init - type = scheme -[do_mynnsfclay] - standard_name = flag_for_mellor_yamada_nakanishi_niino_surface_layer_scheme - long_name = flag to activate MYNN surface layer - units = flag - dimensions = () - type = logical - intent = in -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out - -######################################################################## -[ccpp-arg-table] - name = scheme_3_run - type = scheme -[sfclay_compute_flux] - standard_name = do_compute_surface_scalar_fluxes - long_name = flag for computing surface scalar fluxes in mynnsfclay - units = flag - dimensions = () - type = logical - intent = in -[sfclay_compute_diag] - standard_name = do_compute_surface_diagnostics - long_name = flag for computing surface diagnostics in mynnsfclay - units = flag - dimensions = () - type = logical - intent = in -[prsl] - standard_name = air_pressure - long_name = mean layer pressure - units = Pa - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = inout -[ps] - standard_name = surface_air_pressure - long_name = surface pressure - units = Pa - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[pr_ps] - standard_name = reference_air_pressure_normalized_by_surface_air_pressure - long_name = reference pressure normalized by surface pressure - units = cm - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = out - -######################################################################## -[ccpp-arg-table] - name = scheme_3_timestep_finalize - type = scheme -[prsl] - standard_name = air_pressure - long_name = mean layer pressure - units = Pa - dimensions = (horizontal_dimension,vertical_layer_dimension) - type = real - kind = kind_phys - intent = out -[ps] - standard_name = surface_air_pressure - long_name = surface pressure - units = Pa - dimensions = (horizontal_dimension) - type = real - kind = kind_phys - intent = inout - -######################################################################## -[ccpp-arg-table] - name = scheme_3_timestep_finalize - type = scheme -[prsl] - standard_name = air_pressure - long_name = mean layer pressure - units = Pa - dimensions = (horizontal_dimension,vertical_layer_dimension) - type = real - kind = kind_phys - intent = out -[ps] - standard_name = surface_air_pressure - long_name = surface pressure - units = Pa - dimensions = (horizontal_dimension) - type = real - kind = kind_phys - intent = out - diff --git a/test_prebuild/test_track_variables/scheme_B.meta b/test_prebuild/test_track_variables/scheme_B.meta deleted file mode 100644 index 3bd0f070..00000000 --- a/test_prebuild/test_track_variables/scheme_B.meta +++ /dev/null @@ -1,48 +0,0 @@ -######################################################################## -[ccpp-table-properties] - name = scheme_B - type = scheme - dependencies = ../../hooks/machine.F,module_nst_parameters.f90,module_nst_water_prop.f90 - -######################################################################## -[ccpp-arg-table] - name = scheme_B_run - type = scheme -[im] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent - units = count - dimensions = () - type = integer - intent = in -[wet] - standard_name = flag_nonzero_wet_surface_fraction - long_name = flag indicating presence of some ocean or lake surface area fraction - units = flag - dimensions = (horizontal_loop_extent) - type = logical - intent = in -[tgice] - standard_name = freezing_point_temperature_of_seawater - long_name = freezing point temperature of seawater - units = K - dimensions = () - type = real - kind = kind_phys - intent = in -[tsfco] - standard_name = sea_surface_temperature - long_name = sea surface temperature - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[tsurf_wat] - standard_name = surface_skin_temperature_after_iteration_over_water - long_name = surface skin temperature after iteration over water - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout diff --git a/test_prebuild/test_track_variables/suite_TEST_SUITE.xml b/test_prebuild/test_track_variables/suite_TEST_SUITE.xml deleted file mode 100644 index 92d7af46..00000000 --- a/test_prebuild/test_track_variables/suite_TEST_SUITE.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - - - - scheme_1 - scheme_2 - - - scheme_3 - scheme_4 - - - - - scheme_1 - scheme_A - - - scheme_B - scheme_4 - - - diff --git a/test_prebuild/test_track_variables/suite_small_suite.xml b/test_prebuild/test_track_variables/suite_small_suite.xml deleted file mode 100644 index 057936b3..00000000 --- a/test_prebuild/test_track_variables/suite_small_suite.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - scheme_1 - - - - - scheme_4 - - - diff --git a/test_prebuild/test_unit_conv/CMakeLists.txt b/test_prebuild/test_unit_conv/CMakeLists.txt deleted file mode 100644 index 97c1730c..00000000 --- a/test_prebuild/test_unit_conv/CMakeLists.txt +++ /dev/null @@ -1,98 +0,0 @@ -#------------------------------------------------------------------------------ -cmake_minimum_required(VERSION 3.10) - -project(ccpp_unit_conv - VERSION 1.0.0 - LANGUAGES Fortran) - -#------------------------------------------------------------------------------ -# Request a static build -option(BUILD_SHARED_LIBS "Build a shared library" OFF) - -#------------------------------------------------------------------------------ -# Set MPI flags for C/C++/Fortran with MPI F08 interface -find_package(MPI REQUIRED Fortran) -if(NOT MPI_Fortran_HAVE_F08_MODULE) - message(FATAL_ERROR "MPI implementation does not support the Fortran 2008 mpi_f08 interface") -endif() - -#------------------------------------------------------------------------------ -# Set the sources: physics type definitions -set(TYPEDEFS $ENV{CCPP_TYPEDEFS}) -if(TYPEDEFS) - message(STATUS "Got CCPP TYPEDEFS from environment variable: ${TYPEDEFS}") -else(TYPEDEFS) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_TYPEDEFS.cmake) - message(STATUS "Got CCPP TYPEDEFS from cmakefile include file: ${TYPEDEFS}") -endif(TYPEDEFS) - -# Generate list of Fortran modules from the CCPP type -# definitions that need need to be installed -foreach(typedef_module ${TYPEDEFS}) - list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${typedef_module}) -endforeach() - -#------------------------------------------------------------------------------ -# Set the sources: physics schemes -set(SCHEMES $ENV{CCPP_SCHEMES}) -if(SCHEMES) - message(STATUS "Got CCPP SCHEMES from environment variable: ${SCHEMES}") -else(SCHEMES) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_SCHEMES.cmake) - message(STATUS "Got CCPP SCHEMES from cmakefile include file: ${SCHEMES}") -endif(SCHEMES) - -# Set the sources: physics scheme caps -set(CAPS $ENV{CCPP_CAPS}) -if(CAPS) - message(STATUS "Got CCPP CAPS from environment variable: ${CAPS}") -else(CAPS) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_CAPS.cmake) - message(STATUS "Got CCPP CAPS from cmakefile include file: ${CAPS}") -endif(CAPS) - -# Set the sources: physics scheme caps -set(API $ENV{CCPP_API}) -if(API) - message(STATUS "Got CCPP API from environment variable: ${API}") -else(API) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_API.cmake) - message(STATUS "Got CCPP API from cmakefile include file: ${API}") -endif(API) - -set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O0 -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fbounds-check -ggdb -fbacktrace -ffree-line-length-none") - -#------------------------------------------------------------------------------ -add_library(ccpp_unit_conv STATIC ${SCHEMES} ${CAPS} ${API}) -target_link_libraries(ccpp_unit_conv PRIVATE MPI::MPI_Fortran) -# Generate list of Fortran modules from defined sources -foreach(source_f90 ${CAPS} ${API}) - get_filename_component(tmp_source_f90 ${source_f90} NAME) - string(REGEX REPLACE ".F90" ".mod" tmp_module_f90 ${tmp_source_f90}) - string(TOLOWER ${tmp_module_f90} module_f90) - list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${module_f90}) -endforeach() - -set_target_properties(ccpp_unit_conv PROPERTIES VERSION ${PROJECT_VERSION} - SOVERSION ${PROJECT_VERSION_MAJOR}) - -add_executable(test_unit_conv.x main.F90) -add_dependencies(test_unit_conv.x ccpp_unit_conv) -target_link_libraries(test_unit_conv.x ccpp_unit_conv) -set_target_properties(test_unit_conv.x PROPERTIES LINKER_LANGUAGE Fortran) - -# Define where to install the library -install(TARGETS ccpp_unit_conv - EXPORT ccpp_unit_conv-targets - ARCHIVE DESTINATION lib - LIBRARY DESTINATION lib - RUNTIME DESTINATION lib -) -# Export our configuration -install(EXPORT ccpp_unit_conv-targets - FILE ccpp_unit_conv-config.cmake - DESTINATION lib/cmake -) -# Define where to install the C headers and Fortran modules -#install(FILES ${HEADERS_C} DESTINATION include) -install(FILES ${MODULES_F90} DESTINATION include) diff --git a/test_prebuild/test_unit_conv/ccpp_kinds.F90 b/test_prebuild/test_unit_conv/ccpp_kinds.F90 deleted file mode 100644 index a07ded9b..00000000 --- a/test_prebuild/test_unit_conv/ccpp_kinds.F90 +++ /dev/null @@ -1,13 +0,0 @@ -module ccpp_kinds - - !! \section arg_table_ccpp_kinds - !! \htmlinclude ccpp_kinds.html - !! - - use iso_fortran_env, only: real64 - - implicit none - - integer, parameter :: kind_phys = real64 - -end module ccpp_kinds diff --git a/test_prebuild/test_unit_conv/ccpp_kinds.meta b/test_prebuild/test_unit_conv/ccpp_kinds.meta deleted file mode 100644 index 0e95702e..00000000 --- a/test_prebuild/test_unit_conv/ccpp_kinds.meta +++ /dev/null @@ -1,15 +0,0 @@ -[ccpp-table-properties] - name = ccpp_kinds - type = module - dependencies = - -######################################################################## -[ccpp-arg-table] - name = ccpp_kinds - type = module -[kind_phys] - standard_name = kind_phys - long_name = definition of kind_phys - units = none - dimensions = () - type = integer diff --git a/test_prebuild/test_unit_conv/ccpp_prebuild_config.py b/test_prebuild/test_unit_conv/ccpp_prebuild_config.py deleted file mode 100755 index 3ee45942..00000000 --- a/test_prebuild/test_unit_conv/ccpp_prebuild_config.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python - -# CCPP prebuild config for GFDL Finite-Volume Cubed-Sphere Model (FV3) - -############################################################################### -# Definitions # -############################################################################### - -HOST_MODEL_IDENTIFIER = "FV3" - -# Add all files with metadata tables on the host model side and in CCPP, -# relative to basedir = top-level directory of host model. This includes -# kind and type definitions used in CCPP physics. Also add any internal -# dependencies of these files to the list. -VARIABLE_DEFINITION_FILES = [ - # actual variable definition files - '../../src/ccpp_types.F90', - 'ccpp_kinds.F90', - 'data.F90', - ] - -TYPEDEFS_NEW_METADATA = { - 'ccpp_types' : { - 'ccpp_t' : 'cdata', - 'ccpp_types' : '', - }, - 'data' : { - 'data' : '', - }, - } - -# Add all physics scheme files relative to basedir -SCHEME_FILES = [ - 'unit_conv_scheme_1.F90', - 'unit_conv_scheme_2.F90', - ] - -# Default build dir, relative to current working directory, -# if not specified as command-line argument -DEFAULT_BUILD_DIR = 'build' - -# Auto-generated makefile/cmakefile snippets that contain all type definitions -TYPEDEFS_MAKEFILE = '{build_dir}/CCPP_TYPEDEFS.mk' -TYPEDEFS_CMAKEFILE = '{build_dir}/CCPP_TYPEDEFS.cmake' -TYPEDEFS_SOURCEFILE = '{build_dir}/CCPP_TYPEDEFS.sh' - -# Auto-generated makefile/cmakefile snippets that contain all schemes -SCHEMES_MAKEFILE = '{build_dir}/CCPP_SCHEMES.mk' -SCHEMES_CMAKEFILE = '{build_dir}/CCPP_SCHEMES.cmake' -SCHEMES_SOURCEFILE = '{build_dir}/CCPP_SCHEMES.sh' - -# Auto-generated makefile/cmakefile snippets that contain all caps -CAPS_MAKEFILE = '{build_dir}/CCPP_CAPS.mk' -CAPS_CMAKEFILE = '{build_dir}/CCPP_CAPS.cmake' -CAPS_SOURCEFILE = '{build_dir}/CCPP_CAPS.sh' - -# Directory where to put all auto-generated physics caps -CAPS_DIR = '{build_dir}' - -# Directory where the suite definition files are stored -SUITES_DIR = '.' - -# Optional arguments - only required for schemes that use -# optional arguments. ccpp_prebuild.py will throw an exception -# if it encounters a scheme subroutine with optional arguments -# if no entry is made here. Possible values are: 'all', 'none', -# or a list of standard_names: [ 'var1', 'var3' ]. -OPTIONAL_ARGUMENTS = {} - -# Directory where to write static API to -STATIC_API_DIR = '{build_dir}' -STATIC_API_CMAKEFILE = '{build_dir}/CCPP_API.cmake' -STATIC_API_SOURCEFILE = '{build_dir}/CCPP_API.sh' - -# Directory for writing HTML pages generated from metadata files -METADATA_HTML_OUTPUT_DIR = '{build_dir}' - -# HTML document containing the model-defined CCPP variables -HTML_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_UNIT_CONV.html' - -# LaTeX document containing the provided vs requested CCPP variables -LATEX_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_UNIT_CONV.tex' diff --git a/test_prebuild/test_unit_conv/data.F90 b/test_prebuild/test_unit_conv/data.F90 deleted file mode 100644 index ad6db921..00000000 --- a/test_prebuild/test_unit_conv/data.F90 +++ /dev/null @@ -1,24 +0,0 @@ -module data - - !! \section arg_table_data Argument Table - !! \htmlinclude data.html - !! - use ccpp_kinds, only : kind_phys - use ccpp_types, only: ccpp_t - - implicit none - - private - - public ncols, ncolsrun, nspecies - public cdata, data_array, data_array2, opt_array_flag - - integer, parameter :: ncols = 4 - integer, parameter :: ncolsrun = ncols - integer, parameter :: nspecies = 2 - type(ccpp_t), target :: cdata - real(kind=kind_phys), dimension(1:ncols, 1:nspecies) :: data_array - real(kind=kind_phys), dimension(1:ncols) :: data_array2 - logical :: opt_array_flag - -end module data diff --git a/test_prebuild/test_unit_conv/main.F90 b/test_prebuild/test_unit_conv/main.F90 deleted file mode 100644 index f414eeda..00000000 --- a/test_prebuild/test_unit_conv/main.F90 +++ /dev/null @@ -1,97 +0,0 @@ -program test_unit_conv - - use, intrinsic :: iso_fortran_env, only: error_unit - - use ccpp_types, only: ccpp_t - use data, only: ncols, & - nspecies - use data, only: cdata, & - data_array, & - data_array2, & - opt_array_flag - - use ccpp_static_api, only: ccpp_physics_init, & - ccpp_physics_timestep_init, & - ccpp_physics_run, & - ccpp_physics_timestep_finalize, & - ccpp_physics_finalize - - implicit none - - character(len=*), parameter :: ccpp_suite = 'unit_conv_suite' - integer :: ierr - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - ! For physics running over the entire domain, - ! ccpp_thread_number and ccpp_chunk_number are - ! set to 1, indicating that arrays are to be sent - ! following their dimension specification in the - ! metadata (must match horizontal_dimension). - cdata%thrd_no = 1 - cdata%chunk_no = 1 - cdata%thrd_cnt = 1 - - data_array = 1.0_8 - data_array2 = 42.0_8 - opt_array_flag = .true. - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - call ccpp_physics_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics timestep init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - call ccpp_physics_timestep_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics run step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - call ccpp_physics_run(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_run:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics timestep finalize step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - call ccpp_physics_timestep_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_finalize:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics finalize step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - call ccpp_physics_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - -contains - -end program test_unit_conv diff --git a/test_prebuild/test_unit_conv/run_test.sh b/test_prebuild/test_unit_conv/run_test.sh deleted file mode 100755 index ab7e8c31..00000000 --- a/test_prebuild/test_unit_conv/run_test.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -e - -rm -fr build -mkdir build -../../scripts/ccpp_prebuild.py --verbose --debug --config=ccpp_prebuild_config.py --builddir=build -cd build -cmake .. 2>&1 | tee log.cmake -make VERBOSE=1 -j1 2>&1 | tee log.make -./test_unit_conv.x -cd .. -rm -fr build diff --git a/test_prebuild/test_unit_conv/unit_conv_scheme_1.F90 b/test_prebuild/test_unit_conv/unit_conv_scheme_1.F90 deleted file mode 100644 index 42df267e..00000000 --- a/test_prebuild/test_unit_conv/unit_conv_scheme_1.F90 +++ /dev/null @@ -1,70 +0,0 @@ -!>\file unit_conv_scheme_1.F90 -!! This file contains a unit_conv_scheme_1 CCPP scheme that does nothing -!! except requesting the minimum, mandatory variables. - -module unit_conv_scheme_1 - - use, intrinsic :: iso_fortran_env, only: error_unit - use ccpp_kinds, only : kind_phys - implicit none - - private - public :: unit_conv_scheme_1_run - - !! This is for unit testing only - real(kind=kind_phys), parameter :: target_value = 1.0_kind_phys - real(kind=kind_phys), parameter :: target_value2 = 42.0_kind_phys - -contains - - !! \section arg_table_unit_conv_scheme_1_run Argument Table - !! \htmlinclude unit_conv_scheme_1_run.html - !! - subroutine unit_conv_scheme_1_run(data_array, data_array2, data_array_opt, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - real(kind=kind_phys), intent(inout) :: data_array(:) - real(kind=kind_phys), intent(inout) :: data_array2(:) - real(kind=kind_phys), intent(inout), optional :: data_array_opt(:) - - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Check values in data array - write(error_unit, '(a,e12.4)') & - 'In unit_conv_scheme_1_run: checking min/max values of data array to be approximately ', target_value - if (minval(data_array) < 0.99 * target_value .or. maxval(data_array) > 1.01 * target_value) then - write(errmsg, '(3(a,e12.4),a)') & - "Error in unit_conv_scheme_1_run, expected values for data_array of approximately ", & - target_value, " but got [ ", minval(data_array), " : ", maxval(data_array), " ]" - errflg = 1 - return - end if - ! Check values in data array2 - write(error_unit, '(a,e12.4)') & - 'In unit_conv_scheme_1_run: checking min/max values of data array 2 to be approximately ', target_value2 - if (minval(data_array2) < 0.99 * target_value2 .or. maxval(data_array2) > 1.01 * target_value2) then - write(errmsg, '(3(a,e12.4),a)') & - "Error in unit_conv_scheme_1_run, expected values for data array 2 of approximately ", & - target_value2, " but got [ ", minval(data_array2), " : ", maxval(data_array2), " ]" - errflg = 1 - return - end if - ! Check for presence of optional data array, then check its values - write(error_unit, '(a)') 'In unit_conv_scheme_1_run: checking for presence of optional data array' - if (.not. present(data_array_opt)) then - write(error_unit, '(a)') 'Error in unit_conv_scheme_1_run, optional data array expected but not present' - errflg = 1 - return - end if - write(error_unit, '(a,e12.4)') & - 'In unit_conv_scheme_1_run: checking min/max values of optional data array to be approximately ', target_value - if (minval(data_array_opt) < 0.99 * target_value .or. maxval(data_array_opt) > 1.01 * target_value) then - write(errmsg, '(3(a,e12.4),a)') 'Error in unit_conv_scheme_1_run, expected values of approximately ', & - target_value, ' but got [ ', minval(data_array_opt), ' : ', maxval(data_array_opt), ' ]' - errflg = 1 - return - end if - end subroutine unit_conv_scheme_1_run - -end module unit_conv_scheme_1 diff --git a/test_prebuild/test_unit_conv/unit_conv_scheme_2.F90 b/test_prebuild/test_unit_conv/unit_conv_scheme_2.F90 deleted file mode 100644 index 76f6ef2f..00000000 --- a/test_prebuild/test_unit_conv/unit_conv_scheme_2.F90 +++ /dev/null @@ -1,69 +0,0 @@ -!>\file unit_conv_scheme_2.F90 -!! This file contains a unit_conv_scheme_2 CCPP scheme that does nothing -!! except requesting the minimum, mandatory variables. - -module unit_conv_scheme_2 - - use, intrinsic :: iso_fortran_env, only: error_unit - use ccpp_kinds, only : kind_phys - implicit none - - private - public :: unit_conv_scheme_2_run - - !! This is for unit testing only - real(kind=kind_phys), parameter :: target_value = 1.0E-3_kind_phys - real(kind=kind_phys), parameter :: target_value2 = 42.0_kind_phys - -contains - - !! \section arg_table_unit_conv_scheme_2_run Argument Table - !! \htmlinclude unit_conv_scheme_2_run.html - !! - subroutine unit_conv_scheme_2_run(data_array, data_array2, data_array_opt, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - real(kind=kind_phys), intent(inout) :: data_array(:) - real(kind=kind_phys), intent(inout) :: data_array2(:) - real(kind=kind_phys), intent(inout), optional :: data_array_opt(:) - - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Check values in data array - write(error_unit, '(a,e12.4)') & - 'In unit_conv_scheme_2_run: checking min/max values of data array to be approximately ', target_value - if (minval(data_array) < 0.99 * target_value .or. maxval(data_array) > 1.01 * target_value) then - write(errmsg, '(3(a,e12.4),a)') 'Error in unit_conv_scheme_2_run, expected values of approximately ', & - target_value, ' but got [ ', minval(data_array), ' : ', maxval(data_array), ' ]' - errflg = 1 - return - end if - ! Check values in data array2 - write(error_unit, '(a,e12.4)') & - 'In unit_conv_scheme_2_run: checking min/max values of data array 2 to be approximately ', target_value2 - if (minval(data_array2) < 0.99 * target_value2 .or. maxval(data_array2) > 1.01 * target_value2) then - write(errmsg, '(3(a,e12.4),a)') & - "Error in unit_conv_scheme_2_run, expected values for data array 2 of approximately ", & - target_value2, " but got [ ", minval(data_array2), " : ", maxval(data_array2), " ]" - errflg = 1 - return - end if - ! Check for presence of optional data array, then check its values - write(error_unit, '(a)') 'In unit_conv_scheme_2_run: checking for presence of optional data array' - if (.not. present(data_array_opt)) then - write(error_unit, '(a)') 'Error in unit_conv_scheme_2_run, optional data array expected but not present' - errflg = 1 - return - end if - write(error_unit, '(a,e12.4)') & - 'In unit_conv_scheme_2_run: checking min/max values of optional data array to be approximately ', target_value - if (minval(data_array_opt) < 0.99 * target_value .or. maxval(data_array_opt) > 1.01 * target_value) then - write(errmsg, '(3(a,e12.4),a)') 'Error in unit_conv_scheme_2_run, expected values of approximately ', & - target_value, ' but got [ ', minval(data_array_opt), ' : ', maxval(data_array_opt), ' ]' - errflg = 1 - return - end if - end subroutine unit_conv_scheme_2_run - -end module unit_conv_scheme_2 diff --git a/test_prebuild/unit_tests/run_tests.sh b/test_prebuild/unit_tests/run_tests.sh deleted file mode 100755 index 3b80c071..00000000 --- a/test_prebuild/unit_tests/run_tests.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -THIS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -FRAMEWORK_DIR="${THIS_DIR}/../.." - -export PYTHONPATH="${FRAMEWORK_DIR}/scripts/parse_tools:${FRAMEWORK_DIR}/scripts:${PYTHONPATH}" - -set -ex -python3 ./test_metadata_parser.py -python3 ./test_mkstatic.py diff --git a/test_prebuild/unit_tests/test_metadata_parser.py b/test_prebuild/unit_tests/test_metadata_parser.py deleted file mode 100644 index b4e36dfc..00000000 --- a/test_prebuild/unit_tests/test_metadata_parser.py +++ /dev/null @@ -1,57 +0,0 @@ -import logging -import os -import sys - -from parse_checkers import registered_fortran_ddt_names -from metadata_table import MetadataTable, parse_metadata_file, Var -from framework_env import CCPPFrameworkEnv - -example_table = """ -[ccpp-table-properties] - name = - type = scheme - dependencies_path = path - dependencies = a.f,b.f - -[ccpp-arg-table] - name = - type = scheme -[ im ] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent, start at 1 - units = index - type = integer - dimensions = () - intent = in -""" - - -def test_MetadataTable_parse_table(tmpdir): - path = str(tmpdir.join("table.meta")) - with open(path, "w") as f: - f.write(example_table) - - dummy_run_env = CCPPFrameworkEnv(None, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - - - metadata_headers = parse_metadata_file(path, known_ddts=registered_fortran_ddt_names(), - run_env=dummy_run_env) - - # check metadata header - assert len(metadata_headers) == 1 - metadata_header = metadata_headers[0] - assert metadata_header.table_name == "" - assert metadata_header.table_type == "scheme" - assert metadata_header.dependencies_path == "path" - assert metadata_header.dependencies == [os.path.join(tmpdir, metadata_header.dependencies_path,"a.f"), os.path.join(tmpdir, metadata_header.dependencies_path,"b.f")] - - # check metadata section - assert len(metadata_header.sections()) == 1 - metadata_section = metadata_header.sections()[0] - assert metadata_section.name == "" - assert metadata_section.ptype == "scheme" - (im_data,) = metadata_section.variable_list() - assert isinstance(im_data, Var) - assert im_data.get_dimensions() == [] diff --git a/test_prebuild/unit_tests/test_mkstatic.py b/test_prebuild/unit_tests/test_mkstatic.py deleted file mode 100644 index af66d575..00000000 --- a/test_prebuild/unit_tests/test_mkstatic.py +++ /dev/null @@ -1,22 +0,0 @@ -from mkstatic import extract_parents_and_indices_from_local_name - -import pytest - - -@pytest.mark.parametrize( - "input_,expected", - [ - ( - r"Atm(mytile)%q(:,:,:,Atm2(mytile2)%graupel)", - ("Atm", ["Atm2", "mytile", "mytile2"]), - ), - (r"Atm(mytile)%q(:,:,:,simple_ind)", ("Atm", ["mytile", "simple_ind"])), - (r"Atm%q(random)", ("Atm", ["random"])), - ], -) -def test_extract_parents_and_indices_from_local_name(input_, expected): - expected_parent, expected_inputs = expected - parent, inputs = extract_parents_and_indices_from_local_name(input_) - - assert parent == expected_parent - assert set(inputs) == set(expected_inputs) diff --git a/unit-tests/__init__.py b/unit-tests/__init__.py new file mode 100644 index 00000000..4f9a36ed --- /dev/null +++ b/unit-tests/__init__.py @@ -0,0 +1 @@ +"""Unit and integration tests for ccpp-capgen.""" diff --git a/unit-tests/conftest.py b/unit-tests/conftest.py new file mode 100644 index 00000000..4413cf36 --- /dev/null +++ b/unit-tests/conftest.py @@ -0,0 +1,30 @@ +"""pytest configuration for capgen unit tests. + +Adds the capgen package root to sys.path so that ``import metadata`` and +``import generator`` work regardless of where pytest is invoked from. + +Layout assumed:: + + / + capgen/ <-- the package being tested + unit-tests/ <-- this file's parent directory + conftest.py + test_*.py +""" + +import os +import sys + +# this directory (unit-tests/) — needed so tests can ``from +# import …``. The directory name contains a hyphen so it can't be +# used as a Python module, but adding it to sys.path makes each +# top-level test_*.py importable as a flat module. +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +# repository root (parent of unit-tests/) +_REPO_ROOT = os.path.dirname(_TESTS_DIR) +# capgen/ package directory (sibling of unit-tests/) +_CAPGEN_DIR = os.path.join(_REPO_ROOT, 'capgen') + +for _path in (_TESTS_DIR, _CAPGEN_DIR, _REPO_ROOT): + if _path not in sys.path: + sys.path.insert(0, _path) diff --git a/unit-tests/run_tests.py b/unit-tests/run_tests.py new file mode 100644 index 00000000..12ad7e02 --- /dev/null +++ b/unit-tests/run_tests.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Convenience script to run all capgen unit tests. + +Usage:: + + python unit-tests/run_tests.py # run all tests + python unit-tests/run_tests.py -v # verbose output + python unit-tests/run_tests.py --doctest # include doctests + +This script sets up ``sys.path`` and then delegates to ``unittest.main`` so +that the tests can be run without installing the package. +""" + +import os +import sys +import unittest + +# ---- path setup ------------------------------------------------------------ +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_REPO_ROOT = os.path.dirname(_TESTS_DIR) +_CAPGEN_DIR = os.path.join(_REPO_ROOT, 'capgen') + +for _p in (_CAPGEN_DIR, _REPO_ROOT): + if _p not in sys.path: + sys.path.insert(0, _p) + +# ---- optional --doctest flag ----------------------------------------------- +_include_doctests = '--doctest' in sys.argv +if _include_doctests: + sys.argv.remove('--doctest') + +if __name__ == '__main__': + loader = unittest.TestLoader() + suite = loader.discover(start_dir=_TESTS_DIR, pattern='test_*.py') + + if _include_doctests: + import doctest + import metadata.metadata_table as _mt + suite.addTests(doctest.DocTestSuite(_mt)) + + runner = unittest.TextTestRunner(verbosity=2 if '-v' in sys.argv else 1) + result = runner.run(suite) + sys.exit(0 if result.wasSuccessful() else 1) diff --git a/unit-tests/sample_files/bad_ctrl_in_host_table.meta b/unit-tests/sample_files/bad_ctrl_in_host_table.meta new file mode 100644 index 00000000..4cf16a82 --- /dev/null +++ b/unit-tests/sample_files/bad_ctrl_in_host_table.meta @@ -0,0 +1,17 @@ +# Host table that declares suite_name — it should be in a control table instead. + +[ccpp-table-properties] + name = misplaced_host + type = host + +[ccpp-arg-table] + name = misplaced_host + type = host + +[ suite_name_var ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 diff --git a/unit-tests/sample_files/bad_ctrl_missing_suite_name.meta b/unit-tests/sample_files/bad_ctrl_missing_suite_name.meta new file mode 100644 index 00000000..2554d704 --- /dev/null +++ b/unit-tests/sample_files/bad_ctrl_missing_suite_name.meta @@ -0,0 +1,60 @@ +# Control table with all required vars except suite_name. +# Paired with bad_ctrl_in_host_table.meta to test the "wrong table" error. + +[ccpp-table-properties] + name = ctrl_no_suite_name + type = control + +[ccpp-arg-table] + name = ctrl_no_suite_name + type = control + +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range + units = index + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = physics thread budget + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=256 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer +[ inst_num ] + standard_name = instance_number + long_name = current model instance number + units = 1 + dimensions = () + type = integer diff --git a/unit-tests/sample_files/bad_ctrl_missing_vars.meta b/unit-tests/sample_files/bad_ctrl_missing_vars.meta new file mode 100644 index 00000000..24a7b878 --- /dev/null +++ b/unit-tests/sample_files/bad_ctrl_missing_vars.meta @@ -0,0 +1,18 @@ +# Control table that is missing all required variables except suite_name. +# Used to test that the generator reports all missing vars in one pass. + +[ccpp-table-properties] + name = incomplete_ctrl + type = control + +[ccpp-arg-table] + name = incomplete_ctrl + type = control + +[ suite_name_var ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 diff --git a/unit-tests/sample_files/bad_ctrl_nonscalar.meta b/unit-tests/sample_files/bad_ctrl_nonscalar.meta new file mode 100644 index 00000000..ebfa5b07 --- /dev/null +++ b/unit-tests/sample_files/bad_ctrl_nonscalar.meta @@ -0,0 +1,66 @@ +# Control table where ccpp_error_code has dimensions (must be scalar). + +[ccpp-table-properties] + name = nonscalar_ctrl + type = control + +[ccpp-arg-table] + name = nonscalar_ctrl + type = control + +[ suite_name_var ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range + units = index + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = physics thread budget + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=256 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = (horizontal_dimension) + type = integer +[ inst_num ] + standard_name = instance_number + long_name = current model instance number + units = 1 + dimensions = () + type = integer diff --git a/unit-tests/sample_files/bad_ctrl_wrong_type.meta b/unit-tests/sample_files/bad_ctrl_wrong_type.meta new file mode 100644 index 00000000..f3574687 --- /dev/null +++ b/unit-tests/sample_files/bad_ctrl_wrong_type.meta @@ -0,0 +1,66 @@ +# Control table where horizontal_loop_begin is declared as real instead of integer. + +[ccpp-table-properties] + name = wrong_type_ctrl + type = control + +[ccpp-arg-table] + name = wrong_type_ctrl + type = control + +[ suite_name_var ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range + units = index + dimensions = () + type = real +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range + units = index + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = physics thread budget + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=256 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer +[ inst_num ] + standard_name = instance_number + long_name = current model instance number + units = 1 + dimensions = () + type = integer diff --git a/unit-tests/sample_files/bad_dim_loop_begin.meta b/unit-tests/sample_files/bad_dim_loop_begin.meta new file mode 100644 index 00000000..e7ba7060 --- /dev/null +++ b/unit-tests/sample_files/bad_dim_loop_begin.meta @@ -0,0 +1,33 @@ +# Scheme variable that uses horizontal_loop_begin as a dimension — forbidden. + +[ccpp-table-properties] + name = bad_dim_scheme + type = scheme + +[ccpp-arg-table] + name = bad_dim_scheme_run + type = scheme + +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out +[ data ] + standard_name = some_data + long_name = data array with forbidden dimension + units = 1 + dimensions = (horizontal_loop_begin) + type = real + kind = kind_phys + intent = in diff --git a/unit-tests/sample_files/bad_dim_loop_extent.meta b/unit-tests/sample_files/bad_dim_loop_extent.meta new file mode 100644 index 00000000..1c8cad2b --- /dev/null +++ b/unit-tests/sample_files/bad_dim_loop_extent.meta @@ -0,0 +1,23 @@ +# Host variable that uses horizontal_loop_extent as a dimension — forbidden. + +[ccpp-table-properties] + name = bad_dim_host + type = host + +[ccpp-arg-table] + name = bad_dim_host + type = host + +[ ncols ] + standard_name = horizontal_dimension + long_name = horizontal dimension + units = count + dimensions = () + type = integer +[ data ] + standard_name = some_data + long_name = data array dimensioned by loop extent + units = 1 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys diff --git a/unit-tests/sample_files/bad_duplicate_stdname.meta b/unit-tests/sample_files/bad_duplicate_stdname.meta new file mode 100644 index 00000000..f7e49f41 --- /dev/null +++ b/unit-tests/sample_files/bad_duplicate_stdname.meta @@ -0,0 +1,21 @@ +# Scheme with two variables sharing the same standard name — must be rejected. + +[ccpp-table-properties] + name = dup_scheme + type = scheme + +[ccpp-arg-table] + name = dup_scheme_run + type = scheme +[ a_var ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + intent = in +[ b_var ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + intent = in diff --git a/unit-tests/sample_files/bad_finalize_phase.meta b/unit-tests/sample_files/bad_finalize_phase.meta new file mode 100644 index 00000000..edbc30db --- /dev/null +++ b/unit-tests/sample_files/bad_finalize_phase.meta @@ -0,0 +1,23 @@ +# Scheme with old 'finalize' phase name — must be rejected. +# The generator should emit a hard error directing the user to rename to 'final'. + +[ccpp-table-properties] + name = old_scheme + type = scheme + +[ccpp-arg-table] + name = old_scheme_finalize + type = scheme +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/sample_files/bad_invalid_type.meta b/unit-tests/sample_files/bad_invalid_type.meta new file mode 100644 index 00000000..d728f68e --- /dev/null +++ b/unit-tests/sample_files/bad_invalid_type.meta @@ -0,0 +1,14 @@ +# Table with 'type = banana' — must be rejected with a clear error. + +[ccpp-table-properties] + name = bad_table + type = banana + +[ccpp-arg-table] + name = bad_table + type = banana +[ im ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/bad_module_type.meta b/unit-tests/sample_files/bad_module_type.meta new file mode 100644 index 00000000..1a2ff906 --- /dev/null +++ b/unit-tests/sample_files/bad_module_type.meta @@ -0,0 +1,15 @@ +# This file uses the old 'type = module' which must be rejected. +# The generator should emit a helpful error directing the user to 'type = host'. + +[ccpp-table-properties] + name = physics_module + type = module + +[ccpp-arg-table] + name = physics_module + type = module +[ im ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_chunked_data.meta b/unit-tests/sample_files/control_chunked_data.meta new file mode 100644 index 00000000..e7a4c47c --- /dev/null +++ b/unit-tests/sample_files/control_chunked_data.meta @@ -0,0 +1,79 @@ +# Control metadata for the chunked_data integration test. + +[ccpp-table-properties] + name = chunked_ctrl + type = control + +[ccpp-arg-table] + name = chunked_ctrl + type = control + +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 +[ grp_name ] + standard_name = group_name + long_name = name of the CCPP group + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ thrd_no ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=256 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer +[ inst_no ] + standard_name = instance_number + long_name = current model instance index + units = 1 + dimensions = () + type = integer +[ ninstances ] + standard_name = number_of_instances + long_name = total number of model instances + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_full.meta b/unit-tests/sample_files/control_full.meta new file mode 100644 index 00000000..1bc31c7b --- /dev/null +++ b/unit-tests/sample_files/control_full.meta @@ -0,0 +1,79 @@ +# Full control metadata for generator tests. + +[ccpp-table-properties] + name = host_ctrl + type = control + +[ccpp-arg-table] + name = host_ctrl + type = control + +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ grp_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer +[ inst_num ] + standard_name = instance_number + long_name = current model instance number + units = 1 + dimensions = () + type = integer +[ ninstances ] + standard_name = number_of_instances + long_name = total number of model instances + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_inst_only.meta b/unit-tests/sample_files/control_inst_only.meta new file mode 100644 index 00000000..d796f960 --- /dev/null +++ b/unit-tests/sample_files/control_inst_only.meta @@ -0,0 +1,74 @@ +# Half-paired multi-instance host: control table declares instance_number +# but NOT its mandatory pair partner number_of_instances. Used to +# exercise the paired-validation error path (declaring one of the pair +# without the other must raise). +[ccpp-table-properties] + name = ccpp_control + type = control + +[ccpp-arg-table] + name = ccpp_control + type = control +[ suite_name_var ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 +[ grp_name ] + standard_name = group_name + long_name = name of the CCPP group + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = total number of physics threads + units = 1 + dimensions = () + type = integer +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer +[ inst_num ] + standard_name = instance_number + long_name = current model instance number + units = 1 + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_ninst_only.meta b/unit-tests/sample_files/control_ninst_only.meta new file mode 100644 index 00000000..69402958 --- /dev/null +++ b/unit-tests/sample_files/control_ninst_only.meta @@ -0,0 +1,74 @@ +# Half-paired multi-instance host: control table declares +# number_of_instances but NOT its mandatory pair partner instance_number. +# Used to exercise the paired-validation error path (declaring one of +# the pair without the other must raise). +[ccpp-table-properties] + name = ccpp_control + type = control + +[ccpp-arg-table] + name = ccpp_control + type = control +[ suite_name_var ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 +[ grp_name ] + standard_name = group_name + long_name = name of the CCPP group + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = total number of physics threads + units = 1 + dimensions = () + type = integer +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer +[ ninstances ] + standard_name = number_of_instances + long_name = total number of model instances + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_no_instance.meta b/unit-tests/sample_files/control_no_instance.meta new file mode 100644 index 00000000..282b6a48 --- /dev/null +++ b/unit-tests/sample_files/control_no_instance.meta @@ -0,0 +1,67 @@ +# Single-instance host: control table omits instance_number. The paired +# host metadata (host_no_instance.meta) likewise omits number_of_instances. +# Used to exercise the no-instance-API code paths in the generator. +[ccpp-table-properties] + name = ccpp_control + type = control + +[ccpp-arg-table] + name = ccpp_control + type = control +[ suite_name_var ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 +[ grp_name ] + standard_name = group_name + long_name = name of the CCPP group + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = total number of physics threads + units = 1 + dimensions = () + type = integer +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_opt_arg.meta b/unit-tests/sample_files/control_opt_arg.meta new file mode 100644 index 00000000..27919026 --- /dev/null +++ b/unit-tests/sample_files/control_opt_arg.meta @@ -0,0 +1,79 @@ +# Minimal control metadata for the opt_arg integration test. + +[ccpp-table-properties] + name = opt_arg_ctrl + type = control + +[ccpp-arg-table] + name = opt_arg_ctrl + type = control + +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 +[ grp_name ] + standard_name = group_name + long_name = name of the CCPP group + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ thrd_no ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=256 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer +[ inst_num ] + standard_name = instance_number + long_name = current model instance number + units = 1 + dimensions = () + type = integer +[ ninstances ] + standard_name = number_of_instances + long_name = total number of model instances + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_simple.meta b/unit-tests/sample_files/control_simple.meta new file mode 100644 index 00000000..7b5d2980 --- /dev/null +++ b/unit-tests/sample_files/control_simple.meta @@ -0,0 +1,76 @@ +[ccpp-table-properties] + name = ccpp_control + type = control + +[ccpp-arg-table] + name = ccpp_control + type = control +[ suite_name_var ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 +[ grp_name ] + standard_name = group_name + long_name = name of the CCPP group + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = total number of physics threads + units = 1 + dimensions = () + type = integer +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=256 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer +[ inst_num ] + standard_name = instance_number + long_name = current model instance number + units = 1 + dimensions = () + type = integer +[ ninstances ] + standard_name = number_of_instances + long_name = total number of model instances + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_unit_conv.meta b/unit-tests/sample_files/control_unit_conv.meta new file mode 100644 index 00000000..1d73d541 --- /dev/null +++ b/unit-tests/sample_files/control_unit_conv.meta @@ -0,0 +1,79 @@ +# Control metadata for the unit_conv integration test. + +[ccpp-table-properties] + name = unit_conv_ctrl + type = control + +[ccpp-arg-table] + name = unit_conv_ctrl + type = control + +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 +[ grp_name ] + standard_name = group_name + long_name = name of the CCPP group + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = physics thread budget + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer +[ inst_num ] + standard_name = instance_number + long_name = current model instance number + units = 1 + dimensions = () + type = integer +[ ninstances ] + standard_name = number_of_instances + long_name = total number of model instances + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/ddt_chunked_data.meta b/unit-tests/sample_files/ddt_chunked_data.meta new file mode 100644 index 00000000..0ad6cea9 --- /dev/null +++ b/unit-tests/sample_files/ddt_chunked_data.meta @@ -0,0 +1,16 @@ +# DDT definition for the chunked_data integration test. + +[ccpp-table-properties] + name = chunked_data_type + type = ddt + +[ccpp-arg-table] + name = chunked_data_type + type = ddt + +[ array_data ] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer diff --git a/unit-tests/sample_files/ddt_nested_inner.meta b/unit-tests/sample_files/ddt_nested_inner.meta new file mode 100644 index 00000000..53f8dbe4 --- /dev/null +++ b/unit-tests/sample_files/ddt_nested_inner.meta @@ -0,0 +1,22 @@ +# Inner DDT used as a field inside ddt_nested_outer.meta. + +[ccpp-table-properties] + name = ddt_inner_type + type = ddt + +[ccpp-arg-table] + name = ddt_inner_type + type = ddt +[ inner_value ] + standard_name = inner_real_value + long_name = a real value inside the inner DDT + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys +[ inner_flag ] + standard_name = inner_integer_flag + long_name = an integer flag inside the inner DDT + units = flag + dimensions = () + type = integer diff --git a/unit-tests/sample_files/ddt_nested_outer.meta b/unit-tests/sample_files/ddt_nested_outer.meta new file mode 100644 index 00000000..ad2a4a53 --- /dev/null +++ b/unit-tests/sample_files/ddt_nested_outer.meta @@ -0,0 +1,22 @@ +# Outer DDT whose second field is itself a DDT (tests nested DDT flattening). +# Field 'inner_ddt' is of type 'ddt_inner_type', defined in ddt_nested_inner.meta. + +[ccpp-table-properties] + name = ddt_outer_type + type = ddt + +[ccpp-arg-table] + name = ddt_outer_type + type = ddt +[ scalar_field ] + standard_name = outer_scalar_field + long_name = a scalar field on the outer DDT + units = 1 + dimensions = () + type = integer +[ inner_ddt ] + standard_name = inner_ddt_instance + long_name = nested inner DDT + units = none + dimensions = () + type = ddt_inner_type diff --git a/unit-tests/sample_files/ddt_simple.meta b/unit-tests/sample_files/ddt_simple.meta new file mode 100644 index 00000000..f3731a7c --- /dev/null +++ b/unit-tests/sample_files/ddt_simple.meta @@ -0,0 +1,21 @@ +[ccpp-table-properties] + name = gfs_statein_type + type = ddt + +[ccpp-arg-table] + name = gfs_statein_type + type = ddt +[ phii ] + standard_name = geopotential_at_interface + long_name = geopotential at model layer interfaces + units = m2 s-2 + dimensions = (horizontal_dimension, vertical_interface_dimension) + type = real + kind = kind_phys +[ phil ] + standard_name = geopotential + long_name = geopotential at model layer centers + units = m2 s-2 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys diff --git a/unit-tests/sample_files/ddt_subcycle_stdname.meta b/unit-tests/sample_files/ddt_subcycle_stdname.meta new file mode 100644 index 00000000..8c6d215b --- /dev/null +++ b/unit-tests/sample_files/ddt_subcycle_stdname.meta @@ -0,0 +1,17 @@ +# DDT definition holding the subcycle-count field used by +# suite_subcycle_stdname_ddt.xml. The standard name is exposed via the +# DDT instance declared in host_subcycle_stdname_ddt.meta. + +[ccpp-table-properties] + name = physics_state_subcycle + type = ddt + +[ccpp-arg-table] + name = physics_state_subcycle + type = ddt +[ n_sub ] + standard_name = num_subcycles_for_test + long_name = number of subcycles for the test scheme + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/host_chunked_data.meta b/unit-tests/sample_files/host_chunked_data.meta new file mode 100644 index 00000000..df3eba11 --- /dev/null +++ b/unit-tests/sample_files/host_chunked_data.meta @@ -0,0 +1,25 @@ +# Host metadata for the chunked_data integration test. +# Tests a scheme that accesses a DDT-based data array, called once per +# chunk by a host that manages chunking externally (the current chunk is +# passed as a horizontal range, not a control variable). + +[ccpp-table-properties] + name = chunked_data_mod + type = host + +[ccpp-arg-table] + name = chunked_data_mod + type = host + +[ ncols ] + standard_name = horizontal_dimension + long_name = full horizontal dimension + units = count + dimensions = () + type = integer +[ chunked_data_instance ] + standard_name = chunked_data_type_instance + long_name = instance of chunked_data_type + units = DDT + dimensions = () + type = chunked_data_type diff --git a/unit-tests/sample_files/host_full.meta b/unit-tests/sample_files/host_full.meta new file mode 100644 index 00000000..38d60efa --- /dev/null +++ b/unit-tests/sample_files/host_full.meta @@ -0,0 +1,44 @@ +# Full host metadata for generator tests. +# Provides all variables needed by temp_calc_adjust scheme +# plus extras for dimension resolution. + +[ccpp-table-properties] + name = host_phys + type = host + +[ccpp-arg-table] + name = host_phys + type = host + +[ ncols ] + standard_name = horizontal_dimension + long_name = full horizontal dimension + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + long_name = number of vertical layers + units = count + dimensions = () + type = integer +[ nlevp1 ] + standard_name = vertical_interface_dimension + long_name = number of vertical layer interfaces + units = count + dimensions = () + type = integer +[ dt ] + standard_name = time_step_for_physics + long_name = physics time step + units = s + dimensions = () + type = real + kind = kind_phys +[ gt0 ] + standard_name = air_temperature + long_name = model layer mean temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys diff --git a/unit-tests/sample_files/host_no_instance.meta b/unit-tests/sample_files/host_no_instance.meta new file mode 100644 index 00000000..66186c6f --- /dev/null +++ b/unit-tests/sample_files/host_no_instance.meta @@ -0,0 +1,43 @@ +# Single-instance host data: same vars as host_full.meta minus +# number_of_instances. Pairs with control_no_instance.meta (which +# omits instance_number). +[ccpp-table-properties] + name = host_phys + type = host + +[ccpp-arg-table] + name = host_phys + type = host + +[ ncols ] + standard_name = horizontal_dimension + long_name = full horizontal dimension + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + long_name = number of vertical layers + units = count + dimensions = () + type = integer +[ nlevp1 ] + standard_name = vertical_interface_dimension + long_name = number of vertical layer interfaces + units = count + dimensions = () + type = integer +[ dt ] + standard_name = time_step_for_physics + long_name = physics time step + units = s + dimensions = () + type = real + kind = kind_phys +[ gt0 ] + standard_name = air_temperature + long_name = model layer mean temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys diff --git a/unit-tests/sample_files/host_opt_arg.meta b/unit-tests/sample_files/host_opt_arg.meta new file mode 100644 index 00000000..0de58b65 --- /dev/null +++ b/unit-tests/sample_files/host_opt_arg.meta @@ -0,0 +1,46 @@ +# Host metadata for the opt_arg integration test. +# Provides a mandatory integer array, an optional integer array, +# an optional real array (km; scheme expects m — Case 4), and an +# active flag controlling whether the optional vars are present. + +[ccpp-table-properties] + name = opt_arg_data + type = host + +[ccpp-arg-table] + name = opt_arg_data + type = host + +[ nx ] + standard_name = size_of_std_arg + long_name = size of std_arg array + units = count + dimensions = () + type = integer +[ std_arg ] + standard_name = std_arg + long_name = mandatory integer array + units = 1 + dimensions = (size_of_std_arg) + type = integer +[ opt_arg ] + standard_name = opt_arg + long_name = optional integer array + units = 1 + dimensions = (size_of_std_arg) + type = integer + active = flag_for_opt_arg +[ opt_arg_2 ] + standard_name = opt_arg_2 + long_name = optional real array in km + units = km + dimensions = (size_of_std_arg) + type = real + kind = kind_phys + active = flag_for_opt_arg +[ flag_for_opt_arg ] + standard_name = flag_for_opt_arg + long_name = flag controlling whether optional vars are present + units = flag + dimensions = () + type = logical diff --git a/unit-tests/sample_files/host_simple.meta b/unit-tests/sample_files/host_simple.meta new file mode 100644 index 00000000..baa5c035 --- /dev/null +++ b/unit-tests/sample_files/host_simple.meta @@ -0,0 +1,19 @@ +[ccpp-table-properties] + name = physics_data + type = host + +[ccpp-arg-table] + name = physics_data + type = host +[ ncols ] + standard_name = horizontal_dimension + long_name = horizontal dimension + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + long_name = number of vertical layers + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/host_subcycle_stdname.meta b/unit-tests/sample_files/host_subcycle_stdname.meta new file mode 100644 index 00000000..5bd4bd5d --- /dev/null +++ b/unit-tests/sample_files/host_subcycle_stdname.meta @@ -0,0 +1,18 @@ +# Supplementary host metadata: adds num_subcycles_for_test (used as a +# subcycle loop= bound in suite_subcycle_stdname.xml). Pairs +# with host_full.meta / control_full.meta which provide all the other +# required vars (loop bounds, errflg, errmsg, etc.). + +[ccpp-table-properties] + name = host_phys_subcycle_helper + type = host + +[ccpp-arg-table] + name = host_phys_subcycle_helper + type = host +[ n_sub ] + standard_name = num_subcycles_for_test + long_name = number of subcycles for the test scheme + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/host_subcycle_stdname_ddt.meta b/unit-tests/sample_files/host_subcycle_stdname_ddt.meta new file mode 100644 index 00000000..211eb9e2 --- /dev/null +++ b/unit-tests/sample_files/host_subcycle_stdname_ddt.meta @@ -0,0 +1,19 @@ +# Supplementary host metadata: declares an instance of the +# physics_state_subcycle DDT, which exposes num_subcycles_for_test as a +# component. The subcycle bound in suite_subcycle_stdname_ddt.xml +# resolves to ``phys_state%n_sub`` (access path), not the bare +# component name. + +[ccpp-table-properties] + name = test_host_with_ddt_mod + type = host + +[ccpp-arg-table] + name = test_host_with_ddt_mod + type = host +[ phys_state ] + standard_name = physics_state_subcycle_ddt_instance + long_name = physics state DDT instance carrying the subcycle counter + units = ddt + dimensions = () + type = physics_state_subcycle diff --git a/unit-tests/sample_files/host_unit_conv.meta b/unit-tests/sample_files/host_unit_conv.meta new file mode 100644 index 00000000..07c421d7 --- /dev/null +++ b/unit-tests/sample_files/host_unit_conv.meta @@ -0,0 +1,39 @@ +# Host metadata for the unit_conv integration test. +# Provides horizontal loop variables, data_array in m (mandatory), +# and data_array_opt in m (optional, controlled by flag_for_opt_array). + +[ccpp-table-properties] + name = unit_conv_data + type = host + +[ccpp-arg-table] + name = unit_conv_data + type = host + +[ ncols ] + standard_name = horizontal_dimension + long_name = full horizontal dimension + units = count + dimensions = () + type = integer +[ data_array ] + standard_name = data_array + long_name = mandatory data array in m + units = m + dimensions = (horizontal_dimension) + type = real + kind = kind_phys +[ data_array_opt ] + standard_name = data_array_opt + long_name = optional data array in m + units = m + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + active = flag_for_opt_array +[ flag_for_opt_array ] + standard_name = flag_for_opt_array + long_name = flag controlling whether optional array is present + units = flag + dimensions = () + type = logical diff --git a/unit-tests/sample_files/host_with_constituents.meta b/unit-tests/sample_files/host_with_constituents.meta new file mode 100644 index 00000000..4f9360f2 --- /dev/null +++ b/unit-tests/sample_files/host_with_constituents.meta @@ -0,0 +1,30 @@ +# Host metadata that exposes the constituent object via the type=host table. +# Required when at least one register-phase scheme produces dynamic +# constituents (intent=out type=ccpp_constituent_properties_t). + +[ccpp-table-properties] + name = host_consts + type = host + +[ccpp-arg-table] + name = host_consts + type = host + +[ ncols ] + standard_name = horizontal_dimension + long_name = full horizontal dimension + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + long_name = number of vertical layers + units = count + dimensions = () + type = integer +[ host_consts_obj ] + standard_name = ccpp_model_constituents_object + long_name = host-owned model constituent object + units = none + dimensions = () + type = ccpp_model_constituents_t diff --git a/unit-tests/sample_files/host_with_ddt_instance.meta b/unit-tests/sample_files/host_with_ddt_instance.meta new file mode 100644 index 00000000..60528d16 --- /dev/null +++ b/unit-tests/sample_files/host_with_ddt_instance.meta @@ -0,0 +1,16 @@ +# Host table that declares a DDT instance (tests the DDT instance pattern). +# The type 'gfs_statein_type' references the DDT defined in ddt_simple.meta. + +[ccpp-table-properties] + name = CCPP_data + type = host + +[ccpp-arg-table] + name = CCPP_data + type = host +[ gfs_statein ] + standard_name = gfs_statein + long_name = GFS state-in DDT instance + units = none + dimensions = (number_of_instances) + type = gfs_statein_type diff --git a/unit-tests/sample_files/host_with_dependencies.meta b/unit-tests/sample_files/host_with_dependencies.meta new file mode 100644 index 00000000..ecc26dde --- /dev/null +++ b/unit-tests/sample_files/host_with_dependencies.meta @@ -0,0 +1,47 @@ +# Host metadata declaring file dependencies via the table-properties +# block. Used to verify that host-table dependencies make it into +# datatable.xml's section (regression test for a bug +# where the generator only walked scheme_tables for deps). + +[ccpp-table-properties] + name = host_with_deps + type = host + dependencies_path = /tmp/fake_phys + dependencies = mp/some_mp_params.F90, radiation/some_rad_param.f + dependencies = chemistry/some_chem.F90 + +[ccpp-arg-table] + name = host_with_deps + type = host +[ ncols ] + standard_name = horizontal_dimension + long_name = full horizontal dimension + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + long_name = number of vertical layers + units = count + dimensions = () + type = integer +[ nlevp1 ] + standard_name = vertical_interface_dimension + long_name = number of vertical layer interfaces + units = count + dimensions = () + type = integer +[ dt ] + standard_name = time_step_for_physics + long_name = physics time step + units = s + dimensions = () + type = real + kind = kind_phys +[ gt0 ] + standard_name = air_temperature + long_name = model layer mean temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys diff --git a/unit-tests/sample_files/host_with_nested_ddt.meta b/unit-tests/sample_files/host_with_nested_ddt.meta new file mode 100644 index 00000000..07eab552 --- /dev/null +++ b/unit-tests/sample_files/host_with_nested_ddt.meta @@ -0,0 +1,16 @@ +# Host table with an instance of ddt_outer_type, which itself contains a +# ddt_inner_type field. Used to test recursive DDT flattening. + +[ccpp-table-properties] + name = nested_host_mod + type = host + +[ccpp-arg-table] + name = nested_host_mod + type = host +[ outer_inst ] + standard_name = outer_ddt_instance + long_name = instance of the outer DDT (scalar, no instance dimension) + units = none + dimensions = () + type = ddt_outer_type diff --git a/unit-tests/sample_files/scheme_auto_clone_consumer.meta b/unit-tests/sample_files/scheme_auto_clone_consumer.meta new file mode 100644 index 00000000..625262e1 --- /dev/null +++ b/unit-tests/sample_files/scheme_auto_clone_consumer.meta @@ -0,0 +1,48 @@ +# auto-clone-constituents: fixture for the legacy auto-clone shim +# test. Two is_constituent consumer args (no register-phase source) +# that exercise the full set of legacy attrs the shim accepts: +# default_value, min_value, water_species, mixing_ratio_type. + +[ccpp-table-properties] + name = auto_clone_consumer + type = scheme + +[ccpp-arg-table] + name = auto_clone_consumer_run + type = scheme +[ qv ] + standard_name = water_vapor_specific_humidity + long_name = base constituent water vapor + diagnostic_name = QV + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = inout + advected = .true. + default_value = 1.0e-12 + min_value = 0.0 + water_species = .true. + mixing_ratio_type = wrt_moist +[ qc ] + standard_name = cloud_liquid_dry_mixing_ratio + long_name = cloud liquid water + diagnostic_name = CLDLIQ + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = in + advected = .true. + default_value = 0.0 +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/test_prebuild/test_blocked_data/blocked_data_scheme.meta b/unit-tests/sample_files/scheme_chunked_data.meta similarity index 57% rename from test_prebuild/test_blocked_data/blocked_data_scheme.meta rename to unit-tests/sample_files/scheme_chunked_data.meta index d92b0da6..964d6bb4 100644 --- a/test_prebuild/test_blocked_data/blocked_data_scheme.meta +++ b/unit-tests/sample_files/scheme_chunked_data.meta @@ -1,30 +1,32 @@ +# Scheme metadata for the chunked_data integration test. + [ccpp-table-properties] - name = blocked_data_scheme + name = chunked_data_scheme type = scheme - dependencies = ######################################################################## [ccpp-arg-table] - name = blocked_data_scheme_init + name = chunked_data_scheme_init type = scheme -[errmsg] + +[ errmsg ] standard_name = ccpp_error_message - long_name = error message for error handling in CCPP + long_name = error message units = none dimensions = () type = character kind = len=* intent = out -[errflg] +[ errflg ] standard_name = ccpp_error_code - long_name = error code for error handling in CCPP + long_name = error flag units = 1 dimensions = () type = integer intent = out -[data_array] - standard_name = blocked_data_array - long_name = blocked data array +[ data_array ] + standard_name = chunked_data_array + long_name = chunked data array units = 1 dimensions = (horizontal_dimension) type = integer @@ -32,26 +34,27 @@ ######################################################################## [ccpp-arg-table] - name = blocked_data_scheme_timestep_init + name = chunked_data_scheme_timestep_init type = scheme -[errmsg] + +[ errmsg ] standard_name = ccpp_error_message - long_name = error message for error handling in CCPP + long_name = error message units = none dimensions = () type = character kind = len=* intent = out -[errflg] +[ errflg ] standard_name = ccpp_error_code - long_name = error code for error handling in CCPP + long_name = error flag units = 1 dimensions = () type = integer intent = out -[data_array] - standard_name = blocked_data_array - long_name = blocked data array +[ data_array ] + standard_name = chunked_data_array + long_name = chunked data array units = 1 dimensions = (horizontal_dimension) type = integer @@ -59,60 +62,55 @@ ######################################################################## [ccpp-arg-table] - name = blocked_data_scheme_run + name = chunked_data_scheme_run type = scheme -[errmsg] + +[ errmsg ] standard_name = ccpp_error_message - long_name = error message for error handling in CCPP + long_name = error message units = none dimensions = () type = character kind = len=* intent = out -[errflg] +[ errflg ] standard_name = ccpp_error_code - long_name = error code for error handling in CCPP + long_name = error flag units = 1 dimensions = () type = integer intent = out -[nb] - standard_name = ccpp_block_number - long_name = number of block for explicit data blocking in CCPP - units = index - dimensions = () - type = integer - intent = in -[data_array] - standard_name = blocked_data_array - long_name = blocked data array +[ data_array ] + standard_name = chunked_data_array + long_name = chunked data array units = 1 - dimensions = (horizontal_loop_extent) + dimensions = (horizontal_dimension) type = integer intent = in ######################################################################## [ccpp-arg-table] - name = blocked_data_scheme_timestep_finalize + name = chunked_data_scheme_timestep_final type = scheme -[errmsg] + +[ errmsg ] standard_name = ccpp_error_message - long_name = error message for error handling in CCPP + long_name = error message units = none dimensions = () type = character kind = len=* intent = out -[errflg] +[ errflg ] standard_name = ccpp_error_code - long_name = error code for error handling in CCPP + long_name = error flag units = 1 dimensions = () type = integer intent = out -[data_array] - standard_name = blocked_data_array - long_name = blocked data array +[ data_array ] + standard_name = chunked_data_array + long_name = chunked data array units = 1 dimensions = (horizontal_dimension) type = integer @@ -120,28 +118,28 @@ ######################################################################## [ccpp-arg-table] - name = blocked_data_scheme_finalize + name = chunked_data_scheme_final type = scheme -[errmsg] + +[ errmsg ] standard_name = ccpp_error_message - long_name = error message for error handling in CCPP + long_name = error message units = none dimensions = () type = character kind = len=* intent = out -[errflg] +[ errflg ] standard_name = ccpp_error_code - long_name = error code for error handling in CCPP + long_name = error flag units = 1 dimensions = () type = integer intent = out -[data_array] - standard_name = blocked_data_array - long_name = blocked data array +[ data_array ] + standard_name = chunked_data_array + long_name = chunked data array units = 1 dimensions = (horizontal_dimension) type = integer intent = in - diff --git a/unit-tests/sample_files/scheme_consume_constituent.meta b/unit-tests/sample_files/scheme_consume_constituent.meta new file mode 100644 index 00000000..278e66ad --- /dev/null +++ b/unit-tests/sample_files/scheme_consume_constituent.meta @@ -0,0 +1,41 @@ +# cam-sima-style scheme that consumes a base constituent and produces +# a tendency. No host or earlier-scheme provider for either — both +# are auto-resolved by capgen via the ccpp_constituents / +# ccpp_constituent_tendencies arrays owned by the suite cap. + +[ccpp-table-properties] + name = consume_constituent + type = scheme + +[ccpp-arg-table] + name = consume_constituent_run + type = scheme +[ cldliq ] + standard_name = cloud_liquid_water_mixing_ratio + long_name = base constituent (read-only) + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = in + advected = .true. +[ tend_cldliq ] + standard_name = tendency_of_cloud_liquid_water_mixing_ratio + long_name = tendency of base constituent + units = kg kg-1 s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = out + constituent = .true. +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/sample_files/scheme_interstitial_consumer.meta b/unit-tests/sample_files/scheme_interstitial_consumer.meta new file mode 100644 index 00000000..5dd72b84 --- /dev/null +++ b/unit-tests/sample_files/scheme_interstitial_consumer.meta @@ -0,0 +1,32 @@ +# Scheme that consumes the suite-owned interstitial variable. + +[ccpp-table-properties] + name = interstitial_consumer + type = scheme + +[ccpp-arg-table] + name = interstitial_consumer_run + type = scheme +[ diag_in ] + standard_name = diagnostic_interstitial_field + long_name = suite-owned diagnostic field + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/sample_files/scheme_interstitial_producer.meta b/unit-tests/sample_files/scheme_interstitial_producer.meta new file mode 100644 index 00000000..54bb7490 --- /dev/null +++ b/unit-tests/sample_files/scheme_interstitial_producer.meta @@ -0,0 +1,32 @@ +# Scheme that produces a suite-owned interstitial variable. + +[ccpp-table-properties] + name = interstitial_producer + type = scheme + +[ccpp-arg-table] + name = interstitial_producer_run + type = scheme +[ diag_out ] + standard_name = diagnostic_interstitial_field + long_name = suite-owned diagnostic field + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out diff --git a/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_meta.meta b/unit-tests/sample_files/scheme_module_name_override.meta similarity index 50% rename from test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_meta.meta rename to unit-tests/sample_files/scheme_module_name_override.meta index 92f20ff1..6aa16208 100644 --- a/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_meta.meta +++ b/unit-tests/sample_files/scheme_module_name_override.meta @@ -1,20 +1,24 @@ +# Scheme metadata that declares ``module_name`` in [ccpp-table-properties] +# distinct from the table ``name``. Used to verify capgen emits +# ``use mod_alt_name, only: ...`` (the explicit module name) rather than +# ``use scheme_alt_name``. + [ccpp-table-properties] - name = CCPPeq1_var_missing_in_meta + name = scheme_alt_name type = scheme - -######################################################################## + module_name = mod_alt_name + [ccpp-arg-table] - name = CCPPeq1_var_missing_in_meta_finalize + name = scheme_alt_name_run type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer +[ im ] + standard_name = horizontal_dimension units = count dimensions = () + type = integer intent = in [ errmsg ] standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP units = none dimensions = () type = character @@ -22,7 +26,6 @@ intent = out [ errflg ] standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP units = 1 dimensions = () type = integer diff --git a/unit-tests/sample_files/scheme_multipart.meta b/unit-tests/sample_files/scheme_multipart.meta new file mode 100644 index 00000000..fd340118 --- /dev/null +++ b/unit-tests/sample_files/scheme_multipart.meta @@ -0,0 +1,87 @@ +# Scheme metadata with three phases: init, run, final. +# Tests the redesign phase naming: 'final' (not 'finalize'). + +[ccpp-table-properties] + name = temp_calc_adjust + type = scheme + +[ccpp-arg-table] + name = temp_calc_adjust_init + type = scheme +[ im ] + standard_name = horizontal_dimension + long_name = horizontal dimension + units = count + dimensions = () + type = integer + intent = in +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = temp_calc_adjust_run + type = scheme +[ im ] + standard_name = horizontal_dimension + long_name = horizontal loop extent + units = count + dimensions = () + type = integer + intent = in +[ timestep ] + standard_name = time_step_for_physics + long_name = physics time step + units = s + dimensions = () + type = real + kind = kind_phys + intent = in +[ temp ] + standard_name = air_temperature + long_name = model layer mean temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = temp_calc_adjust_final + type = scheme +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/sample_files/scheme_multipart_correct.F90 b/unit-tests/sample_files/scheme_multipart_correct.F90 new file mode 100644 index 00000000..881427ae --- /dev/null +++ b/unit-tests/sample_files/scheme_multipart_correct.F90 @@ -0,0 +1,35 @@ +module temp_calc_adjust + + implicit none + private + +contains + + subroutine temp_calc_adjust_init(im, errmsg, errflg) + integer, intent(in) :: im + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + errmsg = '' + errflg = 0 + end subroutine temp_calc_adjust_init + + subroutine temp_calc_adjust_run(im, timestep, temp, & + errmsg, errflg) + use ccpp_kinds, only: kind_phys + integer, intent(in) :: im + real(kind=kind_phys), intent(in) :: timestep + real(kind=kind_phys), intent(inout) :: temp(:,:) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + errmsg = '' + errflg = 0 + end subroutine temp_calc_adjust_run + + subroutine temp_calc_adjust_final(errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + errmsg = '' + errflg = 0 + end subroutine temp_calc_adjust_final + +end module temp_calc_adjust diff --git a/unit-tests/sample_files/scheme_multipart_wrong_args.F90 b/unit-tests/sample_files/scheme_multipart_wrong_args.F90 new file mode 100644 index 00000000..043eea28 --- /dev/null +++ b/unit-tests/sample_files/scheme_multipart_wrong_args.F90 @@ -0,0 +1,28 @@ +module temp_calc_adjust + + implicit none + private + +contains + + ! init has wrong arg count: missing errflg + subroutine temp_calc_adjust_init(im, errmsg) + integer, intent(in) :: im + character(len=*), intent(out) :: errmsg + end subroutine temp_calc_adjust_init + + ! run has a renamed arg (tempo instead of temp) + subroutine temp_calc_adjust_run(im, timestep, tempo, errmsg, errflg) + integer, intent(in) :: im + real, intent(in) :: timestep + real, intent(inout) :: tempo(:,:) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + end subroutine temp_calc_adjust_run + + subroutine temp_calc_adjust_final(errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + end subroutine temp_calc_adjust_final + +end module temp_calc_adjust diff --git a/unit-tests/sample_files/scheme_opt_arg.meta b/unit-tests/sample_files/scheme_opt_arg.meta new file mode 100644 index 00000000..af443aac --- /dev/null +++ b/unit-tests/sample_files/scheme_opt_arg.meta @@ -0,0 +1,163 @@ +# Scheme metadata for the opt_arg integration test. +# Tests Case 2 (optional integer, no transform) and Case 4 +# (optional real in m; host has km → unit conversion km→m). + +[ccpp-table-properties] + name = opt_arg_scheme + type = scheme + +######################################################################## +[ccpp-arg-table] + name = opt_arg_scheme_timestep_init + type = scheme + +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out +[ nx ] + standard_name = size_of_std_arg + long_name = size of std_arg array + units = count + dimensions = () + type = integer + intent = in +[ var ] + standard_name = std_arg + long_name = mandatory integer array + units = 1 + dimensions = (size_of_std_arg) + type = integer + intent = in +[ opt_var ] + standard_name = opt_arg + long_name = optional integer array + units = 1 + dimensions = (size_of_std_arg) + type = integer + intent = out + optional = True +[ opt_var_2 ] + standard_name = opt_arg_2 + long_name = optional real array in m + units = m + dimensions = (size_of_std_arg) + type = real + kind = kind_phys + intent = out + optional = True + +######################################################################## +[ccpp-arg-table] + name = opt_arg_scheme_run + type = scheme + +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out +[ nx ] + standard_name = size_of_std_arg + long_name = size of std_arg array + units = count + dimensions = () + type = integer + intent = in +[ var ] + standard_name = std_arg + long_name = mandatory integer array + units = 1 + dimensions = (size_of_std_arg) + type = integer + intent = inout +[ opt_var ] + standard_name = opt_arg + long_name = optional integer array + units = 1 + dimensions = (size_of_std_arg) + type = integer + intent = inout + optional = True +[ opt_var_2 ] + standard_name = opt_arg_2 + long_name = optional real array in m + units = m + dimensions = (size_of_std_arg) + type = real + kind = kind_phys + intent = inout + optional = True + +######################################################################## +[ccpp-arg-table] + name = opt_arg_scheme_timestep_final + type = scheme + +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out +[ nx ] + standard_name = size_of_std_arg + long_name = size of std_arg array + units = count + dimensions = () + type = integer + intent = in +[ var ] + standard_name = std_arg + long_name = mandatory integer array + units = 1 + dimensions = (size_of_std_arg) + type = integer + intent = inout +[ opt_var ] + standard_name = opt_arg + long_name = optional integer array + units = 1 + dimensions = (size_of_std_arg) + type = integer + intent = in + optional = True +[ opt_var_2 ] + standard_name = opt_arg_2 + long_name = optional real array in m + units = m + dimensions = (size_of_std_arg) + type = real + kind = kind_phys + intent = inout + optional = True diff --git a/unit-tests/sample_files/scheme_register_constituents.meta b/unit-tests/sample_files/scheme_register_constituents.meta new file mode 100644 index 00000000..6dd9e6a1 --- /dev/null +++ b/unit-tests/sample_files/scheme_register_constituents.meta @@ -0,0 +1,33 @@ +# Scheme that registers dynamic constituents during ``_register``. +# The register-phase intent=out arg is an allocatable array of +# ccpp_constituent_properties_t, the special type the suite cap detects +# and feeds through the two-pass merge into the host's +# ccpp_model_constituents_object. + +[ccpp-table-properties] + name = register_constituents + type = scheme + +[ccpp-arg-table] + name = register_constituents_register + type = scheme +[ dyn_const ] + standard_name = dynamic_constituents_for_register_test + long_name = per-scheme constituent array + units = none + dimensions = (:) + type = ccpp_constituent_properties_t + intent = out +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/sample_files/scheme_register_dim_consumer.meta b/unit-tests/sample_files/scheme_register_dim_consumer.meta new file mode 100644 index 00000000..8826e739 --- /dev/null +++ b/unit-tests/sample_files/scheme_register_dim_consumer.meta @@ -0,0 +1,31 @@ +# Scheme that uses the suite-owned scalar dimension produced by +# register_dim_producer's _register phase to dimension a suite-owned +# interstitial array. + +[ccpp-table-properties] + name = register_dim_consumer + type = scheme + +[ccpp-arg-table] + name = register_dim_consumer_run + type = scheme +[ interstitial_var ] + standard_name = output_only_interstitial_variable + long_name = suite-owned interstitial dimensioned by register-set scalar + units = 1 + dimensions = (dimension_for_interstitial_variable) + type = integer + intent = out +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/sample_files/scheme_register_dim_producer.meta b/unit-tests/sample_files/scheme_register_dim_producer.meta new file mode 100644 index 00000000..03479ba2 --- /dev/null +++ b/unit-tests/sample_files/scheme_register_dim_producer.meta @@ -0,0 +1,32 @@ +# Scheme that declares a suite-owned scalar dimension during _register. +# Mirrors the temp_calc_adjust_register pattern: an integer scalar set in +# the register phase that is later used as the upper bound of an +# interstitial array in another scheme's run phase. + +[ccpp-table-properties] + name = register_dim_producer + type = scheme + +[ccpp-arg-table] + name = register_dim_producer_register + type = scheme +[ dim_inter ] + standard_name = dimension_for_interstitial_variable + long_name = size of suite-owned interstitial dimension + units = count + dimensions = () + type = integer + intent = out +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/test/unit_tests/sample_scheme_files/missing_arg_table.meta b/unit-tests/sample_files/scheme_suite_init_final.meta similarity index 55% rename from test/unit_tests/sample_scheme_files/missing_arg_table.meta rename to unit-tests/sample_files/scheme_suite_init_final.meta index f4b920b2..346b62b2 100644 --- a/test/unit_tests/sample_scheme_files/missing_arg_table.meta +++ b/unit-tests/sample_files/scheme_suite_init_final.meta @@ -1,40 +1,41 @@ +# Scheme used for the suite-level / integration test. +# Declares minimal init and final phases with only the standard error +# args, so the resolved suite-init/-final calls don't pull in any +# host-side state and the generated cap stays compact. + [ccpp-table-properties] - name = missing_arg_table + name = suite_init_final_scheme type = scheme - -######################################################################## + [ccpp-arg-table] - name = missing_arg_table_init + name = suite_init_final_scheme_init type = scheme -[ errmsg ] +[errmsg] standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP units = none dimensions = () type = character kind = len=512 intent = out -[ errflg ] +[errflg] standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP units = 1 dimensions = () type = integer intent = out + [ccpp-arg-table] - name = missing_arg_table_finalize + name = suite_init_final_scheme_final type = scheme -[ errmsg ] +[errmsg] standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP units = none dimensions = () type = character kind = len=512 intent = out -[ errflg ] +[errflg] standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP units = 1 dimensions = () type = integer diff --git a/unit-tests/sample_files/scheme_top_at_one.meta b/unit-tests/sample_files/scheme_top_at_one.meta new file mode 100644 index 00000000..578dc1a9 --- /dev/null +++ b/unit-tests/sample_files/scheme_top_at_one.meta @@ -0,0 +1,34 @@ +# Scheme that requests air_temperature with top_at_one = True. +# Pairs with host_full.meta (which declares air_temperature with the +# default top_at_one = False), so the resolver should emit a vertical-flip +# transform on the host-side subscript. + +[ccpp-table-properties] + name = top_at_one_scheme + type = scheme + +[ccpp-arg-table] + name = top_at_one_scheme_run + type = scheme +[ temp ] + standard_name = air_temperature + long_name = model layer mean temperature (top-down) + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout + top_at_one = True +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/sample_files/scheme_unit_conv_1.meta b/unit-tests/sample_files/scheme_unit_conv_1.meta new file mode 100644 index 00000000..726c7562 --- /dev/null +++ b/unit-tests/sample_files/scheme_unit_conv_1.meta @@ -0,0 +1,45 @@ +# Scheme 1 for the unit_conv integration test. +# Expects data_array in m (no conversion — Case 1) and +# data_array_opt in m (optional, no conversion — Case 2). + +[ccpp-table-properties] + name = unit_conv_scheme_1 + type = scheme + +######################################################################## +[ccpp-arg-table] + name = unit_conv_scheme_1_run + type = scheme + +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out +[ data_array ] + standard_name = data_array + long_name = data array in m + units = m + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout +[ data_array_opt ] + standard_name = data_array_opt + long_name = optional data array in m + units = m + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout + optional = True diff --git a/unit-tests/sample_files/scheme_unit_conv_2.meta b/unit-tests/sample_files/scheme_unit_conv_2.meta new file mode 100644 index 00000000..1f50f5ad --- /dev/null +++ b/unit-tests/sample_files/scheme_unit_conv_2.meta @@ -0,0 +1,45 @@ +# Scheme 2 for the unit_conv integration test. +# Expects data_array in km (unit conversion m→km — Case 3) and +# data_array_opt in km (optional + unit conversion — Case 4). + +[ccpp-table-properties] + name = unit_conv_scheme_2 + type = scheme + +######################################################################## +[ccpp-arg-table] + name = unit_conv_scheme_2_run + type = scheme + +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out +[ data_array ] + standard_name = data_array + long_name = data array in km + units = km + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout +[ data_array_opt ] + standard_name = data_array_opt + long_name = optional data array in km + units = km + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout + optional = True diff --git a/test/unit_tests/sample_suite_files/another_suite.xml b/unit-tests/sample_suite_files/another_suite.xml similarity index 100% rename from test/unit_tests/sample_suite_files/another_suite.xml rename to unit-tests/sample_suite_files/another_suite.xml diff --git a/test/unit_tests/sample_suite_files/another_suite2.xml b/unit-tests/sample_suite_files/another_suite2.xml similarity index 100% rename from test/unit_tests/sample_suite_files/another_suite2.xml rename to unit-tests/sample_suite_files/another_suite2.xml diff --git a/test/unit_tests/sample_suite_files/nested_full_suite.xml b/unit-tests/sample_suite_files/nested_full_suite.xml similarity index 100% rename from test/unit_tests/sample_suite_files/nested_full_suite.xml rename to unit-tests/sample_suite_files/nested_full_suite.xml diff --git a/test/unit_tests/sample_suite_files/subsuite1.xml b/unit-tests/sample_suite_files/subsuite1.xml similarity index 100% rename from test/unit_tests/sample_suite_files/subsuite1.xml rename to unit-tests/sample_suite_files/subsuite1.xml diff --git a/test/unit_tests/sample_suite_files/subsuite_inline.xml b/unit-tests/sample_suite_files/subsuite_inline.xml similarity index 100% rename from test/unit_tests/sample_suite_files/subsuite_inline.xml rename to unit-tests/sample_suite_files/subsuite_inline.xml diff --git a/unit-tests/sample_suite_files/suite_auto_clone.xml b/unit-tests/sample_suite_files/suite_auto_clone.xml new file mode 100644 index 00000000..4f0959a2 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_auto_clone.xml @@ -0,0 +1,9 @@ + + + + + auto_clone_consumer + + diff --git a/test/unit_tests/sample_suite_files/suite_bad_v2_duplicate_group.xml b/unit-tests/sample_suite_files/suite_bad_v2_duplicate_group.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_bad_v2_duplicate_group.xml rename to unit-tests/sample_suite_files/suite_bad_v2_duplicate_group.xml diff --git a/test/unit_tests/sample_suite_files/suite_bad_v2_suite_tag.xml b/unit-tests/sample_suite_files/suite_bad_v2_suite_tag.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_bad_v2_suite_tag.xml rename to unit-tests/sample_suite_files/suite_bad_v2_suite_tag.xml diff --git a/test/unit_tests/sample_suite_files/suite_bad_version01.xml b/unit-tests/sample_suite_files/suite_bad_version01.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_bad_version01.xml rename to unit-tests/sample_suite_files/suite_bad_version01.xml diff --git a/test/unit_tests/sample_suite_files/suite_bad_version02.xml b/unit-tests/sample_suite_files/suite_bad_version02.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_bad_version02.xml rename to unit-tests/sample_suite_files/suite_bad_version02.xml diff --git a/test/unit_tests/sample_suite_files/suite_bad_version03.xml b/unit-tests/sample_suite_files/suite_bad_version03.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_bad_version03.xml rename to unit-tests/sample_suite_files/suite_bad_version03.xml diff --git a/test/unit_tests/sample_suite_files/suite_bad_version04.xml b/unit-tests/sample_suite_files/suite_bad_version04.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_bad_version04.xml rename to unit-tests/sample_suite_files/suite_bad_version04.xml diff --git a/unit-tests/sample_suite_files/suite_chunked_data.xml b/unit-tests/sample_suite_files/suite_chunked_data.xml new file mode 100644 index 00000000..731fb6dd --- /dev/null +++ b/unit-tests/sample_suite_files/suite_chunked_data.xml @@ -0,0 +1,9 @@ + + + + + + chunked_data_scheme + + + diff --git a/unit-tests/sample_suite_files/suite_consume_constituent.xml b/unit-tests/sample_suite_files/suite_consume_constituent.xml new file mode 100644 index 00000000..3c0297cb --- /dev/null +++ b/unit-tests/sample_suite_files/suite_consume_constituent.xml @@ -0,0 +1,6 @@ + + + + consume_constituent + + diff --git a/test/unit_tests/sample_suite_files/suite_good_v1_test01.xml b/unit-tests/sample_suite_files/suite_good_v1_test01.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_good_v1_test01.xml rename to unit-tests/sample_suite_files/suite_good_v1_test01.xml diff --git a/test/unit_tests/sample_suite_files/suite_good_v1_test02.xml b/unit-tests/sample_suite_files/suite_good_v1_test02.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_good_v1_test02.xml rename to unit-tests/sample_suite_files/suite_good_v1_test02.xml diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test01.xml b/unit-tests/sample_suite_files/suite_good_v2_test01.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_good_v2_test01.xml rename to unit-tests/sample_suite_files/suite_good_v2_test01.xml diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test01_exp.xml b/unit-tests/sample_suite_files/suite_good_v2_test01_exp.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_good_v2_test01_exp.xml rename to unit-tests/sample_suite_files/suite_good_v2_test01_exp.xml diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test02.xml b/unit-tests/sample_suite_files/suite_good_v2_test02.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_good_v2_test02.xml rename to unit-tests/sample_suite_files/suite_good_v2_test02.xml diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test02_exp.xml b/unit-tests/sample_suite_files/suite_good_v2_test02_exp.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_good_v2_test02_exp.xml rename to unit-tests/sample_suite_files/suite_good_v2_test02_exp.xml diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test03.xml b/unit-tests/sample_suite_files/suite_good_v2_test03.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_good_v2_test03.xml rename to unit-tests/sample_suite_files/suite_good_v2_test03.xml diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test03_exp.xml b/unit-tests/sample_suite_files/suite_good_v2_test03_exp.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_good_v2_test03_exp.xml rename to unit-tests/sample_suite_files/suite_good_v2_test03_exp.xml diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test04.xml b/unit-tests/sample_suite_files/suite_good_v2_test04.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_good_v2_test04.xml rename to unit-tests/sample_suite_files/suite_good_v2_test04.xml diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test04_exp.xml b/unit-tests/sample_suite_files/suite_good_v2_test04_exp.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_good_v2_test04_exp.xml rename to unit-tests/sample_suite_files/suite_good_v2_test04_exp.xml diff --git a/unit-tests/sample_suite_files/suite_interstitial.xml b/unit-tests/sample_suite_files/suite_interstitial.xml new file mode 100644 index 00000000..2938e9ee --- /dev/null +++ b/unit-tests/sample_suite_files/suite_interstitial.xml @@ -0,0 +1,7 @@ + + + + interstitial_producer + interstitial_consumer + + diff --git a/test/unit_tests/sample_suite_files/suite_invalid_group_fortran_id.xml b/unit-tests/sample_suite_files/suite_invalid_group_fortran_id.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_invalid_group_fortran_id.xml rename to unit-tests/sample_suite_files/suite_invalid_group_fortran_id.xml diff --git a/test/unit_tests/sample_suite_files/suite_invalid_scheme_fortran_id.xml b/unit-tests/sample_suite_files/suite_invalid_scheme_fortran_id.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_invalid_scheme_fortran_id.xml rename to unit-tests/sample_suite_files/suite_invalid_scheme_fortran_id.xml diff --git a/test/unit_tests/sample_suite_files/suite_invalid_suite_fortran_id.xml b/unit-tests/sample_suite_files/suite_invalid_suite_fortran_id.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_invalid_suite_fortran_id.xml rename to unit-tests/sample_suite_files/suite_invalid_suite_fortran_id.xml diff --git a/test/unit_tests/sample_suite_files/suite_missing_file.xml b/unit-tests/sample_suite_files/suite_missing_file.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_missing_file.xml rename to unit-tests/sample_suite_files/suite_missing_file.xml diff --git a/test/unit_tests/sample_suite_files/suite_missing_group.xml b/unit-tests/sample_suite_files/suite_missing_group.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_missing_group.xml rename to unit-tests/sample_suite_files/suite_missing_group.xml diff --git a/test/unit_tests/sample_suite_files/suite_missing_loaded_suite.xml b/unit-tests/sample_suite_files/suite_missing_loaded_suite.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_missing_loaded_suite.xml rename to unit-tests/sample_suite_files/suite_missing_loaded_suite.xml diff --git a/test/unit_tests/sample_suite_files/suite_missing_version.xml b/unit-tests/sample_suite_files/suite_missing_version.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_missing_version.xml rename to unit-tests/sample_suite_files/suite_missing_version.xml diff --git a/unit-tests/sample_suite_files/suite_module_name_override.xml b/unit-tests/sample_suite_files/suite_module_name_override.xml new file mode 100644 index 00000000..6df5b815 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_module_name_override.xml @@ -0,0 +1,6 @@ + + + + scheme_alt_name + + diff --git a/unit-tests/sample_suite_files/suite_nested_subcycle.xml b/unit-tests/sample_suite_files/suite_nested_subcycle.xml new file mode 100644 index 00000000..7fc99bdd --- /dev/null +++ b/unit-tests/sample_suite_files/suite_nested_subcycle.xml @@ -0,0 +1,10 @@ + + + + + + temp_calc_adjust + + + + diff --git a/unit-tests/sample_suite_files/suite_opt_arg.xml b/unit-tests/sample_suite_files/suite_opt_arg.xml new file mode 100644 index 00000000..496c6fbf --- /dev/null +++ b/unit-tests/sample_suite_files/suite_opt_arg.xml @@ -0,0 +1,9 @@ + + + + + + opt_arg_scheme + + + diff --git a/test/unit_tests/sample_suite_files/suite_recurse_level2.xml b/unit-tests/sample_suite_files/suite_recurse_level2.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_recurse_level2.xml rename to unit-tests/sample_suite_files/suite_recurse_level2.xml diff --git a/test/unit_tests/sample_suite_files/suite_recurse_level2a.xml b/unit-tests/sample_suite_files/suite_recurse_level2a.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_recurse_level2a.xml rename to unit-tests/sample_suite_files/suite_recurse_level2a.xml diff --git a/test/unit_tests/sample_suite_files/suite_recurse_level3.xml b/unit-tests/sample_suite_files/suite_recurse_level3.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_recurse_level3.xml rename to unit-tests/sample_suite_files/suite_recurse_level3.xml diff --git a/test/unit_tests/sample_suite_files/suite_recurse_level3a.xml b/unit-tests/sample_suite_files/suite_recurse_level3a.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_recurse_level3a.xml rename to unit-tests/sample_suite_files/suite_recurse_level3a.xml diff --git a/test/unit_tests/sample_suite_files/suite_recurse_top1.xml b/unit-tests/sample_suite_files/suite_recurse_top1.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_recurse_top1.xml rename to unit-tests/sample_suite_files/suite_recurse_top1.xml diff --git a/test/unit_tests/sample_suite_files/suite_recurse_top2.xml b/unit-tests/sample_suite_files/suite_recurse_top2.xml similarity index 100% rename from test/unit_tests/sample_suite_files/suite_recurse_top2.xml rename to unit-tests/sample_suite_files/suite_recurse_top2.xml diff --git a/unit-tests/sample_suite_files/suite_register_constituents.xml b/unit-tests/sample_suite_files/suite_register_constituents.xml new file mode 100644 index 00000000..de03eb38 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_register_constituents.xml @@ -0,0 +1,6 @@ + + + + register_constituents + + diff --git a/unit-tests/sample_suite_files/suite_register_dim.xml b/unit-tests/sample_suite_files/suite_register_dim.xml new file mode 100644 index 00000000..0fc85811 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_register_dim.xml @@ -0,0 +1,7 @@ + + + + register_dim_producer + register_dim_consumer + + diff --git a/unit-tests/sample_suite_files/suite_subcycle_stdname.xml b/unit-tests/sample_suite_files/suite_subcycle_stdname.xml new file mode 100644 index 00000000..0f2472a5 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_subcycle_stdname.xml @@ -0,0 +1,8 @@ + + + + + temp_calc_adjust + + + diff --git a/unit-tests/sample_suite_files/suite_subcycle_stdname_ddt.xml b/unit-tests/sample_suite_files/suite_subcycle_stdname_ddt.xml new file mode 100644 index 00000000..1c39345d --- /dev/null +++ b/unit-tests/sample_suite_files/suite_subcycle_stdname_ddt.xml @@ -0,0 +1,8 @@ + + + + + temp_calc_adjust + + + diff --git a/unit-tests/sample_suite_files/suite_test_simple.xml b/unit-tests/sample_suite_files/suite_test_simple.xml new file mode 100644 index 00000000..9a8319ae --- /dev/null +++ b/unit-tests/sample_suite_files/suite_test_simple.xml @@ -0,0 +1,6 @@ + + + + temp_calc_adjust + + diff --git a/unit-tests/sample_suite_files/suite_test_subcycle.xml b/unit-tests/sample_suite_files/suite_test_subcycle.xml new file mode 100644 index 00000000..d7d9a6ba --- /dev/null +++ b/unit-tests/sample_suite_files/suite_test_subcycle.xml @@ -0,0 +1,8 @@ + + + + + temp_calc_adjust + + + diff --git a/unit-tests/sample_suite_files/suite_top_at_one.xml b/unit-tests/sample_suite_files/suite_top_at_one.xml new file mode 100644 index 00000000..cc67b1b5 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_top_at_one.xml @@ -0,0 +1,6 @@ + + + + top_at_one_scheme + + diff --git a/unit-tests/sample_suite_files/suite_unit_conv.xml b/unit-tests/sample_suite_files/suite_unit_conv.xml new file mode 100644 index 00000000..f59cf22a --- /dev/null +++ b/unit-tests/sample_suite_files/suite_unit_conv.xml @@ -0,0 +1,10 @@ + + + + + + unit_conv_scheme_1 + unit_conv_scheme_2 + + + diff --git a/unit-tests/sample_suite_files/suite_with_init_final.xml b/unit-tests/sample_suite_files/suite_with_init_final.xml new file mode 100644 index 00000000..8f11e61a --- /dev/null +++ b/unit-tests/sample_suite_files/suite_with_init_final.xml @@ -0,0 +1,8 @@ + + + suite_init_final_scheme + + temp_calc_adjust + + suite_init_final_scheme + diff --git a/unit-tests/test_auto_clone_constituents.py b/unit-tests/test_auto_clone_constituents.py new file mode 100644 index 00000000..5c378d2a --- /dev/null +++ b/unit-tests/test_auto_clone_constituents.py @@ -0,0 +1,693 @@ +"""Tests for the transient auto-clone-constituents legacy shim. + +This whole file is part of the auto-clone-constituents shim and should +be deleted alongside ``metadata/auto_clone_constituents.py`` when the +shim is retired. Search ``auto-clone-constituents`` to find every +touchpoint. +""" + +from __future__ import annotations + +import doctest +import io +import logging +import os +import sys +import unittest + +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_CAPGEN_DIR = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen') +if _CAPGEN_DIR not in sys.path: + sys.path.insert(0, _CAPGEN_DIR) + +from metadata import auto_clone_constituents # noqa: E402 +from metadata.metadata_table import ( # noqa: E402 + MetaVar, parse_metadata_file, _parse_lines, +) +from metadata.parse_tools import CCPPError, ParseContext # noqa: E402 +from generator.suite_resolver import ( # noqa: E402 + AutoCloneEntry, + _make_auto_clone_entry, + _collect_auto_clone_entries, + _vertical_dim_of, + _synthesised_long_name_from_std, +) +from generator.suite_cap import ( # noqa: E402 + _emit_auto_clone_instantiate, + _fmt_kind_phys_real, + _esc_fortran_char, +) + + +_SAMPLES_DIR = os.path.join(_TESTS_DIR, 'sample_files') + + +def _sf(name): + return os.path.join(_SAMPLES_DIR, name) + + +def _ctx(): + return ParseContext(linenum=1, filename='auto_clone_test.meta') + + +class _AutoCloneFixture(unittest.TestCase): + """Mixin that flips the shim on for the duration of a test and + guarantees it goes back off afterwards (the flag is process state). + """ + + def setUp(self): + # Sanity: never start a test with the flag set by an earlier + # test that crashed before cleanup. + auto_clone_constituents.disable() + + def tearDown(self): + auto_clone_constituents.disable() + + +######################################################################## +# Module surface +######################################################################## + +class TestExtraKnownAttrs(_AutoCloneFixture): + + def test_empty_when_disabled(self): + self.assertFalse(auto_clone_constituents.is_enabled()) + self.assertEqual(auto_clone_constituents.extra_known_attrs(), + frozenset()) + + def test_populated_when_enabled(self): + auto_clone_constituents.enable(_stream=io.StringIO()) + extra = auto_clone_constituents.extra_known_attrs() + self.assertEqual( + extra, + frozenset({ + 'default_value', 'min_value', + 'water_species', 'mixing_ratio_type', + }), + ) + + +class TestEnableDisable(_AutoCloneFixture): + + def test_enable_flips_flag_and_writes_banner(self): + sink = io.StringIO() + auto_clone_constituents.enable(_stream=sink) + self.assertTrue(auto_clone_constituents.is_enabled()) + out = sink.getvalue() + self.assertIn('LEGACY AUTO-CLONE-CONSTITUENTS ENABLED', out) + for attr in ('default_value', 'min_value', + 'water_species', 'mixing_ratio_type'): + self.assertIn(attr, out) + self.assertIn('single-instance', out) + self.assertIn('TRANSIENT', out) + self.assertIn('REMOVED', out) + self.assertGreaterEqual(out.count('*' * 10), 2) + + def test_enable_is_idempotent(self): + sink1 = io.StringIO() + auto_clone_constituents.enable(_stream=sink1) + first = sink1.getvalue() + sink2 = io.StringIO() + auto_clone_constituents.enable(_stream=sink2) + self.assertEqual(sink2.getvalue(), '') + self.assertIn('AUTO-CLONE', first) + + def test_disable_resets(self): + auto_clone_constituents.enable(_stream=io.StringIO()) + self.assertTrue(auto_clone_constituents.is_enabled()) + auto_clone_constituents.disable() + self.assertFalse(auto_clone_constituents.is_enabled()) + + def test_logger_receives_warning(self): + logger = logging.getLogger('auto_clone_test_logger') + records = [] + + class _Capture(logging.Handler): + def emit(self, record): + records.append(record) + + handler = _Capture(level=logging.WARNING) + logger.addHandler(handler) + try: + auto_clone_constituents.enable( + logger=logger, _stream=io.StringIO()) + self.assertEqual(len(records), 1) + self.assertEqual(records[0].levelno, logging.WARNING) + msg = records[0].getMessage() + for attr in ('default_value', 'min_value', + 'water_species', 'mixing_ratio_type'): + self.assertIn(attr, msg) + finally: + logger.removeHandler(handler) + + +class TestSingleInstanceGuard(_AutoCloneFixture): + + def test_noop_when_disabled(self): + # Even a multi-instance host should not raise when the shim is off. + host_dict = {'instance_number': object(), + 'number_of_instances': object()} + auto_clone_constituents.require_single_instance_host(host_dict) # no raise + + def test_single_instance_passes(self): + auto_clone_constituents.enable(_stream=io.StringIO()) + # No instance pair → fine. + auto_clone_constituents.require_single_instance_host({}) + + def test_instance_number_alone_raises(self): + auto_clone_constituents.enable(_stream=io.StringIO()) + with self.assertRaises(CCPPError) as cm: + auto_clone_constituents.require_single_instance_host( + {'instance_number': object()} + ) + self.assertIn('single-instance', str(cm.exception)) + self.assertIn('instance_number', str(cm.exception)) + + def test_number_of_instances_alone_raises(self): + auto_clone_constituents.enable(_stream=io.StringIO()) + with self.assertRaises(CCPPError) as cm: + auto_clone_constituents.require_single_instance_host( + {'number_of_instances': object()} + ) + self.assertIn('number_of_instances', str(cm.exception)) + + +######################################################################## +# Parser: conditional _KNOWN_ATTRS extension +######################################################################## + +class TestParserShimOff(_AutoCloneFixture): + """When the shim is OFF, the four legacy attrs are rejected with + the standard "Unknown variable attribute" error.""" + + def _make_var(self): + return MetaVar('q', _ctx()) + + def test_default_value_rejected(self): + v = self._make_var() + with self.assertRaises(CCPPError) as cm: + v.set_attr('default_value', '0.0', _ctx()) + self.assertIn('Unknown variable attribute', str(cm.exception)) + self.assertIn('default_value', str(cm.exception)) + + def test_min_value_rejected(self): + v = self._make_var() + with self.assertRaises(CCPPError): + v.set_attr('min_value', '0.0', _ctx()) + + def test_water_species_rejected(self): + v = self._make_var() + with self.assertRaises(CCPPError): + v.set_attr('water_species', 'True', _ctx()) + + def test_mixing_ratio_type_rejected(self): + v = self._make_var() + with self.assertRaises(CCPPError): + v.set_attr('mixing_ratio_type', 'dry', _ctx()) + + +class TestParserShimOn(_AutoCloneFixture): + """When the shim is ON, the four legacy attrs parse and validate.""" + + def setUp(self): + super().setUp() + auto_clone_constituents.enable(_stream=io.StringIO()) + + def _make_var(self): + return MetaVar('q', _ctx()) + + def test_default_value_accepted(self): + v = self._make_var() + v.set_attr('default_value', '1.5e-12', _ctx()) + self.assertEqual(v.default_value, 1.5e-12) + + def test_default_value_kind_phys_suffix_accepted(self): + v = self._make_var() + v.set_attr('default_value', '0.0_kind_phys', _ctx()) + self.assertEqual(v.default_value, 0.0) + + def test_default_value_d_exponent_accepted(self): + v = self._make_var() + v.set_attr('default_value', '1.0d-5', _ctx()) + self.assertEqual(v.default_value, 1.0e-5) + + def test_default_value_invalid_raises(self): + v = self._make_var() + with self.assertRaises(CCPPError): + v.set_attr('default_value', 'not_a_number', _ctx()) + + def test_min_value_accepted(self): + v = self._make_var() + v.set_attr('min_value', '-3.14', _ctx()) + self.assertEqual(v.min_value, -3.14) + + def test_min_value_kind_suffix_accepted(self): + v = self._make_var() + v.set_attr('min_value', '1.0e-12_kind_dyn', _ctx()) + self.assertEqual(v.min_value, 1.0e-12) + + def test_water_species_true(self): + v = self._make_var() + v.set_attr('water_species', '.true.', _ctx()) + self.assertIs(v.water_species, True) + + def test_water_species_false(self): + v = self._make_var() + v.set_attr('water_species', 'False', _ctx()) + self.assertIs(v.water_species, False) + + def test_water_species_invalid_raises(self): + v = self._make_var() + with self.assertRaises(CCPPError): + v.set_attr('water_species', 'maybe', _ctx()) + + def test_mixing_ratio_type_dry(self): + v = self._make_var() + v.set_attr('mixing_ratio_type', 'dry', _ctx()) + self.assertEqual(v.mixing_ratio_type, 'dry') + + def test_mixing_ratio_type_lowercased(self): + v = self._make_var() + v.set_attr('mixing_ratio_type', 'WRT_MOIST', _ctx()) + self.assertEqual(v.mixing_ratio_type, 'wrt_moist') + + def test_mixing_ratio_type_invalid_raises(self): + v = self._make_var() + with self.assertRaises(CCPPError): + v.set_attr('mixing_ratio_type', 'bogus', _ctx()) + + +class TestParserSchemeOnly(_AutoCloneFixture): + """The four legacy attrs are scheme-only when the shim is on; the + parser must reject them on host/control/ddt tables.""" + + def setUp(self): + super().setUp() + auto_clone_constituents.enable(_stream=io.StringIO()) + + def test_default_value_rejected_on_host_table(self): + src = ( + '[ccpp-table-properties]\n' + ' name = h\n' + ' type = host\n' + '[ccpp-arg-table]\n' + ' name = h\n' + ' type = host\n' + '[ q ]\n' + ' standard_name = some_field\n' + ' units = kg kg-1\n' + ' dimensions = ()\n' + ' type = real | kind = kind_phys\n' + ' default_value = 0.0\n' + ) + with self.assertRaises(CCPPError) as cm: + _parse_lines(src.splitlines(keepends=True), 't.meta') + self.assertIn('scheme-only', str(cm.exception)) + + +######################################################################## +# Suite-cap helpers +######################################################################## + +class TestKindPhysFormatter(unittest.TestCase): + + def test_positive(self): + self.assertTrue(_fmt_kind_phys_real(0.0).endswith('_kind_phys')) + self.assertIn('e', _fmt_kind_phys_real(1.0e-12)) + + def test_negative(self): + s = _fmt_kind_phys_real(-3.14) + self.assertTrue(s.endswith('_kind_phys')) + self.assertTrue(s.startswith('-')) + + +class TestFortranCharEscape(unittest.TestCase): + + def test_no_quote_passthrough(self): + self.assertEqual(_esc_fortran_char("CLDLIQ"), "CLDLIQ") + + def test_single_quote_doubled(self): + self.assertEqual(_esc_fortran_char("don't"), "don''t") + + +class TestEmitAutoCloneInstantiate(unittest.TestCase): + """Render one synthesised %instantiate call and check the lines.""" + + def _entry(self, **kw): + defaults = dict( + std_name='cloud_liquid_dry_mixing_ratio', + long_name='cloud liquid water', + diag_name='CLDLIQ', + units='kg kg-1', + vertical_dim='vertical_layer_dimension', + advected=False, + molar_mass=0.0, + default_value=None, + min_value=None, + water_species=None, + mixing_ratio_type=None, + ) + defaults.update(kw) + return AutoCloneEntry(**defaults) + + def _emit(self, entry): + lines = [] + _emit_auto_clone_instantiate( + entry, buf='suite_dynamic_constituents', + inst_idx='1', indent=' ', + errflg_local='errflg', errmsg_local='errmsg', + lines=lines, + ) + return lines + + def test_required_kwargs_always_emitted(self): + lines = self._emit(self._entry()) + joined = '\n'.join(lines) + self.assertIn("std_name = 'cloud_liquid_dry_mixing_ratio'", joined) + self.assertIn("long_name = 'cloud liquid water'", joined) + self.assertIn("diag_name = 'CLDLIQ'", joined) + self.assertIn("units = 'kg kg-1'", joined) + self.assertIn("vertical_dim = 'vertical_layer_dimension'", joined) + self.assertIn("errcode = errflg", joined) + self.assertIn("errmsg = errmsg", joined) + # Error guard. + self.assertIn('if (errflg /= 0) return', joined) + # Counter increment. + self.assertEqual(lines[0].strip(), 'num_consts = num_consts + 1') + + def test_optional_kwargs_omitted_when_unset(self): + lines = self._emit(self._entry()) + joined = '\n'.join(lines) + for kw in ('advected', 'default_value', 'min_value', + 'water_species', 'mixing_ratio_type', 'molar_mass'): + self.assertNotIn(kw + ' ', joined, + "unset optional '{}' leaked into output".format(kw)) + + def test_advected_only_emitted_when_true(self): + lines = self._emit(self._entry(advected=True)) + joined = '\n'.join(lines) + self.assertIn('advected = .true.', joined) + + def test_real_kwargs_emit_kind_phys_literal(self): + lines = self._emit(self._entry( + default_value=0.0, min_value=1.0e-12, molar_mass=18.015, + )) + joined = '\n'.join(lines) + self.assertIn('default_value=', joined) + self.assertIn('min_value =', joined) + self.assertIn('molar_mass =', joined) + # Each real kwarg uses the kind_phys suffix. + self.assertEqual(joined.count('_kind_phys'), 3) + + def test_water_species_true_and_false(self): + on = self._emit(self._entry(water_species=True)) + off = self._emit(self._entry(water_species=False)) + self.assertIn('water_species= .true.', '\n'.join(on)) + self.assertIn('water_species= .false.', '\n'.join(off)) + + def test_mixing_ratio_type_quoted(self): + lines = self._emit(self._entry(mixing_ratio_type='wrt_moist')) + self.assertIn("mixing_ratio_type = 'wrt_moist'", '\n'.join(lines)) + + def test_quoted_long_name_escaped(self): + # Fortran character literal escape: every single quote is + # doubled. + lines = self._emit(self._entry(long_name="don't")) + self.assertIn("long_name = 'don''t'", '\n'.join(lines)) + + +######################################################################## +# Resolver helpers +######################################################################## + +class TestVerticalDimOf(unittest.TestCase): + + class _StubVar: + def __init__(self, dims): + self.dimensions = dims + + def test_extracts_layer_dim(self): + v = self._StubVar(['horizontal_dimension', 'vertical_layer_dimension']) + self.assertEqual(_vertical_dim_of(v), 'vertical_layer_dimension') + + def test_extracts_interface_dim(self): + v = self._StubVar( + ['horizontal_dimension', 'vertical_interface_dimension']) + self.assertEqual(_vertical_dim_of(v), 'vertical_interface_dimension') + + def test_no_vertical_dim_returns_default(self): + v = self._StubVar(['horizontal_dimension']) + self.assertEqual(_vertical_dim_of(v), 'vertical_layer_dimension') + + def test_explicit_range_form_supported(self): + v = self._StubVar([ + 'ccpp_constant_one:horizontal_dimension', + 'ccpp_constant_one:vertical_layer_dimension', + ]) + self.assertEqual(_vertical_dim_of(v), 'vertical_layer_dimension') + + +class TestMakeAutoCloneEntry(_AutoCloneFixture): + """``_make_auto_clone_entry`` snapshots a scheme MetaVar.""" + + def setUp(self): + super().setUp() + auto_clone_constituents.enable(_stream=io.StringIO()) + + def _scheme_var(self, **set_attrs): + ctx = _ctx() + v = MetaVar('qv', ctx) + v.set_attr('standard_name', 'water_vapor_specific_humidity', ctx) + v.set_attr('long_name', 'water vapor', ctx) + v.set_attr('units', 'kg kg-1', ctx) + v.set_attr('dimensions', + '(horizontal_dimension, vertical_layer_dimension)', ctx) + v.set_attr('type', 'real', ctx) + v.set_attr('kind', 'kind_phys', ctx) + v.set_attr('intent', 'inout', ctx) + for k, val in set_attrs.items(): + v.set_attr(k, val, ctx) + return v + + def test_diag_name_defaults_to_local_name(self): + v = self._scheme_var() + entry = _make_auto_clone_entry(v) + self.assertEqual(entry.diag_name, 'qv') # MetaVar.diagnostic_name fallback + + def test_explicit_diagnostic_name_wins(self): + v = self._scheme_var(diagnostic_name='QV') + entry = _make_auto_clone_entry(v) + self.assertEqual(entry.diag_name, 'QV') + + def test_optional_kwargs_passthrough(self): + v = self._scheme_var( + advected='.true.', + molar_mass='18.015', + default_value='1.0e-12', + min_value='0.0', + water_species='.true.', + mixing_ratio_type='wrt_moist', + ) + entry = _make_auto_clone_entry(v) + self.assertTrue(entry.advected) + self.assertAlmostEqual(entry.molar_mass, 18.015) + self.assertEqual(entry.default_value, 1.0e-12) + self.assertEqual(entry.min_value, 0.0) + self.assertIs(entry.water_species, True) + self.assertEqual(entry.mixing_ratio_type, 'wrt_moist') + + def test_vertical_dim_lifted_from_dimensions(self): + v = self._scheme_var() + entry = _make_auto_clone_entry(v) + self.assertEqual(entry.vertical_dim, 'vertical_layer_dimension') + + def test_explicit_long_name_wins(self): + # When the metadata supplies a long_name, the entry carries it + # verbatim (no synthesis). The shared _scheme_var helper + # already sets long_name='water vapor', so we re-use that + # fixture as the "explicit long_name set" case. + v = self._scheme_var() + entry = _make_auto_clone_entry(v) + self.assertEqual(entry.long_name, 'water vapor') + + def test_long_name_synthesised_when_missing(self): + # The _scheme_var helper sets long_name='water vapor'; override + # to a no-op so the helper sees an empty long_name. capgen + # then synthesises from std_name: 'water_vapor_specific_humidity' + # → 'Water vapor specific humidity'. + from metadata.metadata_table import MetaVar + ctx = _ctx() + v = MetaVar('qv', ctx) + v.set_attr('standard_name', + 'water_vapor_specific_humidity', ctx) + v.set_attr('units', 'kg kg-1', ctx) + v.set_attr('dimensions', + '(horizontal_dimension, vertical_layer_dimension)', ctx) + v.set_attr('type', 'real', ctx) + v.set_attr('kind', 'kind_phys', ctx) + v.set_attr('intent', 'inout', ctx) + # No long_name attribute set. + entry = _make_auto_clone_entry(v) + self.assertEqual(entry.long_name, 'Water vapor specific humidity') + + +class TestSynthesisedLongName(unittest.TestCase): + """The ``_synthesised_long_name_from_std`` helper mirrors original + capgen's behaviour: replace each underscore with a space and + capitalise the first character.""" + + def test_typical_constituent(self): + self.assertEqual( + _synthesised_long_name_from_std('cloud_liquid_dry_mixing_ratio'), + 'Cloud liquid dry mixing ratio', + ) + + def test_single_underscore(self): + self.assertEqual( + _synthesised_long_name_from_std('specific_humidity'), + 'Specific humidity', + ) + + def test_no_underscore(self): + self.assertEqual( + _synthesised_long_name_from_std('temperature'), + 'Temperature', + ) + + def test_mixed_case_lowercased_after_first(self): + # Python's ``.capitalize()`` lowercases everything after the + # first character. That's a deliberate match for original + # capgen's behaviour and means a std_name with embedded + # uppercase (atypical) gets normalised. + self.assertEqual( + _synthesised_long_name_from_std('Foo_Bar'), + 'Foo bar', + ) + + +class TestCollectAutoCloneEntriesSkips(_AutoCloneFixture): + """``_collect_auto_clone_entries`` must skip framework-named std + names (``ccpp_constituents``, ``ccpp_constituent_tendencies``, + ``ccpp_constituent_properties``, ``number_of_ccpp_constituents``, + ``index_of_*``) — those resolve through ``source='constituent'`` + too but reference the framework-provided buffers, not individual + species, so synthesising a ``%instantiate`` for them would + register bogus duplicate constituents (e.g. an entry named + ``ccpp_constituents`` in the dynamic-constituents buffer). + """ + + def _resolved_arg(self, std_name): + from generator.suite_resolver import ResolvedArg + return ResolvedArg( + standard_name=std_name, + scheme_local_name='dummy', + intent='inout', + is_optional=False, + active='', active_local='', + source='constituent', + host_entry=None, suite_var=None, + base_expr='', subscript='', call_expr='', + used_dim_std_names=set(), + needs_unit_transform=False, + needs_kind_transform=False, + unit_forward='', unit_backward='', + kind_scheme='', kind_host='', + temp_name='', ptr_name='', + transform_case=1, + scheme_dimensions=[], + ) + + def _resolved_groups_with(self, std_names): + from generator.suite_resolver import ( + ResolvedCall, ResolvedGroup, + ) + call = ResolvedCall( + scheme_name='dummy_scheme', phase='run', + args=[self._resolved_arg(s) for s in std_names], + ) + group = ResolvedGroup(group_name='g') + group.phase_calls['run'] = [call] + return [group] + + def test_framework_array_names_skipped(self): + auto_clone_constituents.enable(_stream=io.StringIO()) + groups = self._resolved_groups_with([ + 'ccpp_constituents', + 'ccpp_constituent_tendencies', + 'ccpp_constituent_properties', + 'number_of_ccpp_constituents', + ]) + # No matching scheme metadata exists, but the framework-name + # filter must fire BEFORE the scheme-var lookup so this never + # gets that far. Returns [] without raising. + entries = _collect_auto_clone_entries(groups, scheme_store=None) + self.assertEqual(entries, []) + + def test_index_of_names_skipped(self): + auto_clone_constituents.enable(_stream=io.StringIO()) + groups = self._resolved_groups_with([ + 'index_of_water_vapor_specific_humidity', + 'index_of_cloud_liquid_dry_mixing_ratio', + ]) + entries = _collect_auto_clone_entries(groups, scheme_store=None) + self.assertEqual(entries, []) + + +######################################################################## +# Integration: full parse + resolver pass on the auto-clone fixture +######################################################################## + +class TestFixtureParse(_AutoCloneFixture): + """The sample fixture parses cleanly with the shim on and carries + every legacy attr on the right scheme args.""" + + def setUp(self): + super().setUp() + auto_clone_constituents.enable(_stream=io.StringIO()) + + def test_fixture_parses(self): + tables = parse_metadata_file(_sf('scheme_auto_clone_consumer.meta')) + self.assertEqual(len(tables), 1) + section = tables[0].sections()[0] + by_name = {v.local_name: v for v in section.variables} + self.assertIn('qv', by_name) + self.assertIn('qc', by_name) + # qv carries the full legacy attr set. + qv = by_name['qv'] + self.assertTrue(qv.is_constituent) + self.assertTrue(qv.advected) + self.assertEqual(qv.default_value, 1.0e-12) + self.assertEqual(qv.min_value, 0.0) + self.assertIs(qv.water_species, True) + self.assertEqual(qv.mixing_ratio_type, 'wrt_moist') + # qc only sets default_value. + qc = by_name['qc'] + self.assertEqual(qc.default_value, 0.0) + self.assertIsNone(qc.min_value) + self.assertIsNone(qc.water_species) + self.assertIsNone(qc.mixing_ratio_type) + + def test_fixture_rejected_without_shim(self): + # Drop the flag; the same .meta file now fails. + auto_clone_constituents.disable() + with self.assertRaises(CCPPError) as cm: + parse_metadata_file(_sf('scheme_auto_clone_consumer.meta')) + # The first rejected attr is ``default_value`` (qv's first + # legacy attr in declaration order). Any of the four would + # be acceptable; check the generic "Unknown variable + # attribute" wording. + self.assertIn('Unknown variable attribute', str(cm.exception)) + + + +######################################################################## +# Doctest loader +######################################################################## + +def load_tests(loader, tests, ignore): + tests.addTests(doctest.DocTestSuite(auto_clone_constituents)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_ccpp_datafile.py b/unit-tests/test_ccpp_datafile.py new file mode 100644 index 00000000..fe03ae98 --- /dev/null +++ b/unit-tests/test_ccpp_datafile.py @@ -0,0 +1,363 @@ +"""Tests for the ccpp_datafile query CLI. + +Covers each of the 17 CLI flags end-to-end by: + 1. building a real datatable.xml via the writer in generator.datatable, + 2. invoking datatable_report / datatable_pretty_print on it, + 3. asserting the textual output. +""" + +import doctest +import os +import shutil +import sys +import tempfile +import unittest + +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_CAPGEN_DIR = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen') +for _p in (_CAPGEN_DIR, _TESTS_DIR): + if _p not in sys.path: + sys.path.insert(0, _p) + +import ccpp_datafile as cdf +from ccpp_datafile import ( + DatatableReport, + datatable_pretty_print, + datatable_report, +) +from generator.datatable import write_datatable + +from test_suite_resolver import ( + _load_full_host_dict, + _load_scheme_store, + _parse_suite, +) +from generator.suite_resolver import resolve_suite + + +def _build_datatable(tmpdir, + host_file_paths=None, utility_paths=None, + suite_file_paths=None, scheme_file_paths=None, + dependency_paths=None, + suite_meta_paths=None, expanded_sdf_paths=None, + protect_first_host_var=False): + """Build a real datatable.xml in *tmpdir* and return its path.""" + hd = _load_full_host_dict() + if protect_first_host_var: + first = next(iter(hd)) + hd[first].protected = True + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + suite_resolution = resolve_suite(suite, store, hd) + return write_datatable( + [suite_resolution], + store, + utility_paths or ['/out/ccpp_kinds.F90'], + suite_file_paths or ['/out/ccpp_test_simple_cap.F90', + '/out/ccpp_test_simple_physics_cap.F90'], + tmpdir, + host_file_paths=host_file_paths or ['/out/test_host_ccpp_cap.F90'], + scheme_file_paths=scheme_file_paths, + dependency_paths=dependency_paths or [], + suite_meta_paths=suite_meta_paths, + expanded_sdf_paths=expanded_sdf_paths, + host_dict=hd, + ) + + +class _DTBase(unittest.TestCase): + """Shared fixture: build one datatable.xml per test class.""" + + @classmethod + def setUpClass(cls): + cls._tmpdir = tempfile.mkdtemp() + cls._datatable = _build_datatable(cls._tmpdir) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls._tmpdir) + + +class TestDatatableReportFileActions(_DTBase): + + def test_host_files(self): + out = datatable_report(self._datatable, + DatatableReport('host_files'), ',') + self.assertEqual(out, '/out/test_host_ccpp_cap.F90') + + def test_suite_files(self): + out = datatable_report(self._datatable, + DatatableReport('suite_files'), ',') + items = out.split(',') + self.assertIn('/out/ccpp_test_simple_cap.F90', items) + self.assertIn('/out/ccpp_test_simple_physics_cap.F90', items) + + def test_utility_files(self): + out = datatable_report(self._datatable, + DatatableReport('utility_files'), ',') + self.assertEqual(out, '/out/ccpp_kinds.F90') + + def test_capgen_files_returns_all(self): + out = datatable_report(self._datatable, + DatatableReport('capgen_files'), ',') + items = out.split(',') + self.assertIn('/out/ccpp_kinds.F90', items) + self.assertIn('/out/test_host_ccpp_cap.F90', items) + self.assertIn('/out/ccpp_test_simple_cap.F90', items) + + def test_separator_honored(self): + out = datatable_report(self._datatable, + DatatableReport('suite_files'), ';') + self.assertIn(';', out) + self.assertNotIn(',', out) + + +class TestDatatableReportInspectionFiles(unittest.TestCase): + """--inspection-files returns meta + expanded SDF paths and excludes them + from --capgen-files.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + self._datatable = _build_datatable( + self._tmpdir, + suite_meta_paths=['/out/ccpp_test_simple_data.meta'], + expanded_sdf_paths=['/out/ccpp_test_simple_expanded.xml'], + ) + + def tearDown(self): + shutil.rmtree(self._tmpdir) + + def test_inspection_files_returns_meta_and_expanded(self): + out = datatable_report(self._datatable, + DatatableReport('inspection_files'), ',') + items = out.split(',') + self.assertIn('/out/ccpp_test_simple_data.meta', items) + self.assertIn('/out/ccpp_test_simple_expanded.xml', items) + + def test_inspection_files_excluded_from_capgen_files(self): + out = datatable_report(self._datatable, + DatatableReport('capgen_files'), ',') + items = out.split(',') + self.assertNotIn('/out/ccpp_test_simple_data.meta', items) + self.assertNotIn('/out/ccpp_test_simple_expanded.xml', items) + + def test_inspection_files_empty_when_none_given(self): + # Build a datatable with no inspection paths; the section is still + # present, and --inspection-files returns an empty string. + with tempfile.TemporaryDirectory() as d: + path = _build_datatable(d) + out = datatable_report(path, + DatatableReport('inspection_files'), ',') + self.assertEqual(out, '') + + +class TestDatatableReportSchemeActions(_DTBase): + + def test_process_list_empty(self): + # capgen does not emit attrs. + out = datatable_report(self._datatable, + DatatableReport('process_list'), ',') + self.assertEqual(out, '') + + def test_module_list_includes_scheme_modules(self): + out = datatable_report(self._datatable, + DatatableReport('module_list'), ',') + modules = out.split(',') + self.assertIn('temp_calc_adjust', modules) + + def test_dependencies_empty_when_none(self): + out = datatable_report(self._datatable, + DatatableReport('dependencies'), ',') + self.assertEqual(out, '') + + +class TestDatatableReportSchemeFiles(unittest.TestCase): + """--scheme-files returns the used-scheme Fortran source paths from + ; the section is always present (possibly empty) so the + query never raises on a vanilla datatable.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self._tmpdir) + + def test_scheme_files_returns_listed_paths(self): + path = _build_datatable( + self._tmpdir, + scheme_file_paths=['/phys/scheme_b.F90', '/phys/scheme_a.F90'], + ) + out = datatable_report(path, DatatableReport('scheme_files'), ',') + items = out.split(',') + # Writer preserves caller order; do not assume sort. + self.assertIn('/phys/scheme_b.F90', items) + self.assertIn('/phys/scheme_a.F90', items) + + def test_scheme_files_empty_when_none_given(self): + path = _build_datatable(self._tmpdir) + out = datatable_report(path, DatatableReport('scheme_files'), ',') + self.assertEqual(out, '') + + +class TestDatatableReportDependenciesPopulated(unittest.TestCase): + """--dependencies returns the sorted, dedup'd dependency list.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + self._datatable = _build_datatable( + self._tmpdir, + dependency_paths=['/dep/b.F90', '/dep/a.F90', '/dep/a.F90'], + ) + + def tearDown(self): + shutil.rmtree(self._tmpdir) + + def test_dependencies_sorted_dedup(self): + out = datatable_report(self._datatable, + DatatableReport('dependencies'), ',') + self.assertEqual(out, '/dep/a.F90,/dep/b.F90') + + +class TestDatatableReportSuiteList(_DTBase): + + def test_suite_list(self): + out = datatable_report(self._datatable, + DatatableReport('suite_list'), ',') + self.assertEqual(out, 'test_simple') + + +class TestDatatableReportVariableActions(_DTBase): + + def test_required_variables_includes_call_list_vars(self): + out = datatable_report( + self._datatable, + DatatableReport('required_variables', 'test_simple'), ',') + names = out.split(',') + self.assertIn('air_temperature', names) + + def test_input_variables_excludes_out_only(self): + # An intent=out var must not appear in --input-variables. + out = datatable_report( + self._datatable, + DatatableReport('input_variables', 'test_simple'), ',') + names = set(out.split(',')) + # ccpp_error_code is intent=out → not in input list. + self.assertNotIn('ccpp_error_code', names) + + def test_output_variables_excludes_in_only(self): + out = datatable_report( + self._datatable, + DatatableReport('output_variables', 'test_simple'), ',') + names = set(out.split(',')) + self.assertIn('ccpp_error_code', names) + + def test_host_variables_returns_host_names(self): + out = datatable_report(self._datatable, + DatatableReport('host_variables'), ',') + names = set(out.split(',')) + self.assertIn('air_temperature', names) + + def test_unknown_suite_returns_empty(self): + out = datatable_report( + self._datatable, + DatatableReport('required_variables', 'no_such_suite'), ',') + self.assertEqual(out, '') + + +class TestExcludeProtected(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + self._datatable = _build_datatable( + self._tmpdir, protect_first_host_var=True, + ) + hd = _load_full_host_dict() + self._protected_name = next(iter(hd)) + + def tearDown(self): + shutil.rmtree(self._tmpdir) + + def test_protected_excluded_from_host_when_flag_on(self): + out = datatable_report( + self._datatable, + DatatableReport('host_variables'), ',', exclude_protected=True) + names = set(out.split(',')) + self.assertNotIn(self._protected_name, names) + + def test_protected_included_when_flag_off(self): + out = datatable_report( + self._datatable, + DatatableReport('host_variables'), ',', exclude_protected=False) + names = set(out.split(',')) + self.assertIn(self._protected_name, names) + + +class TestShowAction(_DTBase): + + def test_show_returns_string(self): + out = datatable_pretty_print(self._datatable, indent=2, line_wrap=-1) + self.assertIsInstance(out, str) + self.assertIn('; it must not now.""" + self.assertIsNone(self._capgen_files().find('suite_meta_files')) + + +class TestInspectionFilesSection(unittest.TestCase): + """ collects non-Fortran artifacts (meta + expanded SDF).""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + path, _, _ = _write( + self._tmpdir, + suite_meta_paths=['/out/ccpp_test_simple_data.meta'], + expanded_sdf_paths=['/out/ccpp_test_simple_expanded.xml'], + ) + self._root = ET.parse(path).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def _inspection(self): + return self._root.find('inspection_files') + + def test_inspection_files_element_present(self): + self.assertIsNotNone(self._inspection()) + + def test_suite_meta_files_subsection(self): + meta = self._inspection().find('suite_meta_files') + self.assertIsNotNone(meta) + files = [f.text for f in meta.findall('file')] + self.assertEqual(files, ['/out/ccpp_test_simple_data.meta']) + + def test_expanded_sdf_files_subsection(self): + exp = self._inspection().find('expanded_sdf_files') + self.assertIsNotNone(exp) + files = [f.text for f in exp.findall('file')] + self.assertEqual(files, ['/out/ccpp_test_simple_expanded.xml']) + + def test_empty_subsections_when_no_paths(self): + """Section + both subsections are always written, even when empty.""" + with tempfile.TemporaryDirectory() as d: + path, _, _ = _write(d) # no meta / expanded paths + root = ET.parse(path).getroot() + insp = root.find('inspection_files') + self.assertIsNotNone(insp) + self.assertIsNotNone(insp.find('suite_meta_files')) + self.assertIsNotNone(insp.find('expanded_sdf_files')) + self.assertEqual(len(insp.find('suite_meta_files').findall('file')), 0) + self.assertEqual(len(insp.find('expanded_sdf_files').findall('file')), 0) + + +class TestSchemesSection(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + path, _, _ = _write(self._tmpdir) + self._root = ET.parse(path).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def _schemes(self): + return self._root.find('schemes') + + def test_schemes_element_present(self): + self.assertIsNotNone(self._schemes()) + + def test_scheme_name_present(self): + names = [s.get('name') for s in self._schemes().findall('scheme')] + self.assertIn('temp_calc_adjust', names) + + def test_no_duplicate_scheme_elements(self): + names = [s.get('name') for s in self._schemes().findall('scheme')] + self.assertEqual(len(names), len(set(names))) + + def test_run_phase_element(self): + scheme = next( + s for s in self._schemes().findall('scheme') + if s.get('name') == 'temp_calc_adjust' + ) + phase_tags = [child.tag for child in scheme] + self.assertIn('run', phase_tags) + + def test_run_phase_attributes(self): + scheme = next( + s for s in self._schemes().findall('scheme') + if s.get('name') == 'temp_calc_adjust' + ) + run_elem = scheme.find('run') + self.assertIsNotNone(run_elem) + self.assertEqual(run_elem.get('name'), 'temp_calc_adjust') + self.assertEqual(run_elem.get('subroutine_name'), 'temp_calc_adjust_run') + self.assertEqual(run_elem.get('module'), 'temp_calc_adjust') + + def test_call_list_present(self): + scheme = next( + s for s in self._schemes().findall('scheme') + if s.get('name') == 'temp_calc_adjust' + ) + run_elem = scheme.find('run') + call_list = run_elem.find('call_list') + self.assertIsNotNone(call_list) + + def test_call_list_has_vars(self): + scheme = next( + s for s in self._schemes().findall('scheme') + if s.get('name') == 'temp_calc_adjust' + ) + run_elem = scheme.find('run') + call_list = run_elem.find('call_list') + vars_ = call_list.findall('var') + self.assertGreater(len(vars_), 0) + + def test_var_has_required_attributes(self): + scheme = next( + s for s in self._schemes().findall('scheme') + if s.get('name') == 'temp_calc_adjust' + ) + run_elem = scheme.find('run') + call_list = run_elem.find('call_list') + for v in call_list.findall('var'): + self.assertIsNotNone(v.get('name'), "var missing 'name'") + self.assertIsNotNone(v.get('intent'), "var missing 'intent'") + self.assertIsNotNone(v.get('local_name'), "var missing 'local_name'") + + def test_error_vars_present_in_call_list(self): + scheme = next( + s for s in self._schemes().findall('scheme') + if s.get('name') == 'temp_calc_adjust' + ) + run_elem = scheme.find('run') + call_list = run_elem.find('call_list') + std_names = [v.get('name') for v in call_list.findall('var')] + self.assertIn('ccpp_error_message', std_names) + self.assertIn('ccpp_error_code', std_names) + + +class TestApiSection(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + path, _, _ = _write(self._tmpdir) + self._root = ET.parse(path).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_api_element_present(self): + self.assertIsNotNone(self._root.find('api')) + + def test_suites_element_present(self): + api = self._root.find('api') + self.assertIsNotNone(api.find('suites')) + + def test_suite_name(self): + suites = self._root.find('api').find('suites') + names = [s.get('name') for s in suites.findall('suite')] + self.assertIn('test_simple', names) + + def test_group_name(self): + suites = self._root.find('api').find('suites') + suite = next(s for s in suites.findall('suite') if s.get('name') == 'test_simple') + groups = suite.findall('group') + self.assertGreater(len(groups), 0) + group_names = [g.get('name') for g in groups] + self.assertIn('physics', group_names) + + def test_scheme_listed_in_group(self): + suites = self._root.find('api').find('suites') + suite = next(s for s in suites.findall('suite') if s.get('name') == 'test_simple') + group = next(g for g in suite.findall('group') if g.get('name') == 'physics') + scheme_names = [s.text for s in group.findall('scheme')] + self.assertIn('temp_calc_adjust', scheme_names) + + def test_no_duplicate_schemes_in_group(self): + suites = self._root.find('api').find('suites') + suite = next(s for s in suites.findall('suite') if s.get('name') == 'test_simple') + group = next(g for g in suite.findall('group') if g.get('name') == 'physics') + names = [s.text for s in group.findall('scheme')] + self.assertEqual(len(names), len(set(names))) + + +class TestDependenciesSection(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + path, _, _ = _write(self._tmpdir) + self._root = ET.parse(path).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_dependencies_element_present(self): + self.assertIsNotNone(self._root.find('dependencies')) + + def test_dependencies_empty_by_default(self): + deps = self._root.find('dependencies') + self.assertEqual(len(list(deps)), 0) + + +class TestDependenciesPopulated(unittest.TestCase): + """write_datatable writes children when paths are given.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + suite_resolution, store = _resolve() + self._path = write_datatable( + [suite_resolution], + store, + [], + [], + self._tmpdir, + dependency_paths=['/path/to/a.F90', '/path/to/b.F90', '/path/to/a.F90'], + ) + self._root = ET.parse(self._path).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_dependency_elements_present(self): + deps = self._root.find('dependencies') + texts = [d.text for d in deps.findall('dependency')] + self.assertIn('/path/to/a.F90', texts) + self.assertIn('/path/to/b.F90', texts) + + def test_dependencies_deduplicated(self): + deps = self._root.find('dependencies') + texts = [d.text for d in deps.findall('dependency')] + self.assertEqual(texts.count('/path/to/a.F90'), 1) + + def test_dependencies_sorted(self): + deps = self._root.find('dependencies') + texts = [d.text for d in deps.findall('dependency')] + self.assertEqual(texts, sorted(texts)) + + +class TestSubcycleDatatable(unittest.TestCase): + """Schemes inside subcycles appear once in the datatable.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + path, _, _ = _write(self._tmpdir, suite_xml='suite_test_subcycle.xml') + self._root = ET.parse(path).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_scheme_deduped_in_schemes(self): + names = [s.get('name') for s in self._root.find('schemes').findall('scheme')] + self.assertEqual(names.count('temp_calc_adjust'), 1) + + def test_scheme_deduped_in_api_group(self): + suites = self._root.find('api').find('suites') + suite = suites.findall('suite')[0] + group = suite.findall('group')[0] + names = [s.text for s in group.findall('scheme')] + self.assertEqual(names.count('temp_calc_adjust'), 1) + + +class TestDiagnosticNameEmission(unittest.TestCase): + """diagnostic_name / diagnostic_name_fixed must reach attributes.""" + + def _find_var(self, root, scheme, phase, std_name): + for s in root.find('schemes').findall('scheme'): + if s.get('name') != scheme: + continue + ph = s.find(phase) + if ph is None: + continue + for v in ph.find('call_list').findall('var'): + if v.get('name') == std_name: + return v + return None + + def test_explicit_diagnostic_name_emitted(self): + with tempfile.TemporaryDirectory() as d: + path, _, store = _write(d) + # Find a scheme variable, set an explicit diagnostic_name on its + # MetaVar, then re-write the datatable. + mvars = store.variables_for('temp_calc_adjust', 'run') + target = next(mv for mv in mvars + if mv.standard_name == 'air_temperature') + target._diagnostic_name = 'temperature' + suite_resolution, _ = _resolve() + path = write_datatable( + [suite_resolution], store, + ['/out/ccpp_kinds.F90'], + ['/out/ccpp_test_simple_cap.F90'], + d, + ) + root = ET.parse(path).getroot() + v = self._find_var(root, 'temp_calc_adjust', 'run', + 'air_temperature') + self.assertIsNotNone(v) + self.assertEqual(v.get('diagnostic_name'), 'temperature') + + def test_diagnostic_name_fixed_emitted(self): + with tempfile.TemporaryDirectory() as d: + suite_resolution, store = _resolve() + mvars = store.variables_for('temp_calc_adjust', 'run') + target = next(mv for mv in mvars + if mv.standard_name == 'air_temperature') + target.diagnostic_name_fixed = 'Q' + path = write_datatable( + [suite_resolution], store, + ['/out/ccpp_kinds.F90'], + ['/out/ccpp_test_simple_cap.F90'], + d, + ) + root = ET.parse(path).getroot() + v = self._find_var(root, 'temp_calc_adjust', 'run', + 'air_temperature') + self.assertEqual(v.get('diagnostic_name_fixed'), 'Q') + # diagnostic_name attr is suppressed (since _fixed wins). + self.assertIsNone(v.get('diagnostic_name')) + + def test_default_diagnostic_name_is_local_name(self): + """No explicit attrs → diagnostic_name attribute equals local_name.""" + with tempfile.TemporaryDirectory() as d: + path, _, _ = _write(d) + root = ET.parse(path).getroot() + v = self._find_var(root, 'temp_calc_adjust', 'run', + 'air_temperature') + self.assertIsNotNone(v) + self.assertEqual(v.get('diagnostic_name'), v.get('local_name')) + + +class TestHostFilesPopulated(unittest.TestCase): + """host_file_paths arg populates .""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + self._path, _, _ = _write( + self._tmpdir, + host_file_paths=['/out/test_host_ccpp_cap.F90'], + ) + self._root = ET.parse(self._path).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_host_files_contains_host_cap(self): + host_files = self._root.find('capgen_files').find('host_files') + names = [f.text for f in host_files.findall('file')] + self.assertEqual(names, ['/out/test_host_ccpp_cap.F90']) + + +class TestVarDictionariesSection(unittest.TestCase): + """write_datatable emits when host_dict is given.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + self._hd = _load_full_host_dict() + self._path, self._sr, _ = _write( + self._tmpdir, + host_dict=self._hd, + ) + self._root = ET.parse(self._path).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def _vd(self): + return self._root.find('var_dictionaries') + + def test_section_emitted_when_host_dict_provided(self): + self.assertIsNotNone(self._vd()) + + def test_no_section_when_host_dict_absent(self): + with tempfile.TemporaryDirectory() as d: + path, _, _ = _write(d) # host_dict=None + root = ET.parse(path).getroot() + self.assertIsNone(root.find('var_dictionaries')) + + def test_host_dictionary_present(self): + host_d = next( + (vd for vd in self._vd().findall('var_dictionary') + if vd.get('type') == 'host'), + None, + ) + self.assertIsNotNone(host_d) + self.assertEqual(host_d.get('name'), 'host') + + def test_host_dictionary_has_vars(self): + host_d = next(vd for vd in self._vd().findall('var_dictionary') + if vd.get('type') == 'host') + names = {v.get('name') for v in host_d.find('variables').findall('var')} + self.assertIn('air_temperature', names) + + def test_api_dict_parent_is_host(self): + api_d = next(vd for vd in self._vd().findall('var_dictionary') + if vd.get('type') == 'api') + # The 'host' string is a fixed internal label written by the + # generator; ccpp_datafile.py uses it only for the api->host walk. + self.assertEqual(api_d.get('parent'), 'host') + + def test_suite_dict_parent_is_api(self): + suite_d = next(vd for vd in self._vd().findall('var_dictionary') + if vd.get('type') == 'suite') + self.assertEqual(suite_d.get('parent'), 'ccpp_api') + self.assertEqual(suite_d.get('name'), 'test_simple') + + def test_group_dict_parent_is_suite(self): + group_d = next(vd for vd in self._vd().findall('var_dictionary') + if vd.get('type') == 'group') + self.assertEqual(group_d.get('parent'), 'test_simple') + + def test_group_call_list_parent_is_group(self): + call_d = next(vd for vd in self._vd().findall('var_dictionary') + if vd.get('type') == 'group_call_list') + gname = call_d.get('name').replace('_call_list', '') + self.assertEqual(call_d.get('parent'), gname) + + def test_group_call_list_has_vars(self): + call_d = next(vd for vd in self._vd().findall('var_dictionary') + if vd.get('type') == 'group_call_list') + vars_ = call_d.find('variables').findall('var') + self.assertGreater(len(vars_), 0) + for v in vars_: + self.assertIsNotNone(v.get('name')) + # intent absent for the rare control-only entry is OK + # but every var must have a name + + +class TestVarDictionariesProtectedAttr(unittest.TestCase): + """Protected host vars carry protected='True'; others omit the attr.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + hd = _load_full_host_dict() + # Flip one entry to protected for the test. + self._protected_std = next(iter(hd)) + hd[self._protected_std].protected = True + self._hd = hd + path, _, _ = _write(self._tmpdir, host_dict=hd) + self._root = ET.parse(path).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_protected_attr_emitted_when_true(self): + host_d = next(vd for vd in self._root.find('var_dictionaries') + .findall('var_dictionary') + if vd.get('type') == 'host') + v = next(v for v in host_d.find('variables').findall('var') + if v.get('name') == self._protected_std) + self.assertEqual(v.get('protected'), 'True') + + def test_protected_attr_absent_when_false(self): + host_d = next(vd for vd in self._root.find('var_dictionaries') + .findall('var_dictionary') + if vd.get('type') == 'host') + non_prot = next(v for v in host_d.find('variables').findall('var') + if v.get('name') != self._protected_std) + self.assertIsNone(non_prot.get('protected')) + + +def load_tests(loader, tests, ignore): + tests.addTests(doctest.DocTestSuite(dt_mod)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_dim_aliases.py b/unit-tests/test_dim_aliases.py new file mode 100644 index 00000000..5c0f50ae --- /dev/null +++ b/unit-tests/test_dim_aliases.py @@ -0,0 +1,223 @@ +"""Tests for the transient GFS dim-aliases shim. + +This whole file is part of the dim-aliases shim and should be deleted +alongside ``metadata/dim_aliases.py`` when the GFS-physics rename is +complete. Search ``dim-aliases`` / ``gfs-dim-aliases`` to find every +touchpoint. +""" + +from __future__ import annotations + +import doctest +import io +import logging +import os +import sys +import unittest + +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_CAPGEN_DIR = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen') +if _CAPGEN_DIR not in sys.path: + sys.path.insert(0, _CAPGEN_DIR) + +from metadata import dim_aliases # noqa: E402 +from generator.suite_resolver import _canonical_dim # noqa: E402 + + +class _DimAliasesFixture(unittest.TestCase): + """Mixin that flips the shim on for the duration of a test and + guarantees it goes back off afterwards (the flag is process state). + """ + + def setUp(self): + # Sanity: never start a test with the flag set by an earlier + # test that crashed before cleanup. + dim_aliases.disable() + + def tearDown(self): + dim_aliases.disable() + + +class TestCanonicalOff(_DimAliasesFixture): + """When the shim is disabled, ``canonical`` is a strict identity.""" + + def test_identity_on_aliased_name_when_disabled(self): + self.assertFalse(dim_aliases.is_enabled()) + self.assertEqual( + dim_aliases.canonical( + 'adjusted_vertical_layer_dimension_for_radiation'), + 'adjusted_vertical_layer_dimension_for_radiation', + ) + self.assertEqual( + dim_aliases.canonical('vertical_composition_dimension'), + 'vertical_composition_dimension', + ) + + def test_identity_on_unknown_name(self): + self.assertEqual( + dim_aliases.canonical('air_temperature'), 'air_temperature', + ) + + +class TestEnableDisable(_DimAliasesFixture): + + def test_enable_flips_flag_and_writes_banner(self): + sink = io.StringIO() + dim_aliases.enable(_stream=sink) + self.assertTrue(dim_aliases.is_enabled()) + out = sink.getvalue() + # Bold banner: starred border, both alias pairs, transient marker. + self.assertIn('GFS DIM-ALIASES ENABLED', out) + self.assertIn('adjusted_vertical_layer_dimension_for_radiation', out) + self.assertIn('vertical_composition_dimension', out) + self.assertIn('vertical_layer_dimension', out) + self.assertIn('TRANSIENT', out) + self.assertIn('REMOVED', out) + self.assertGreaterEqual(out.count('*' * 10), 2) + + def test_enable_is_idempotent(self): + sink1 = io.StringIO() + dim_aliases.enable(_stream=sink1) + first = sink1.getvalue() + sink2 = io.StringIO() + dim_aliases.enable(_stream=sink2) # second call — no banner + self.assertEqual(sink2.getvalue(), '') + self.assertIn('GFS DIM-ALIASES', first) + + def test_disable_resets(self): + dim_aliases.enable(_stream=io.StringIO()) + self.assertTrue(dim_aliases.is_enabled()) + dim_aliases.disable() + self.assertFalse(dim_aliases.is_enabled()) + + def test_logger_receives_warning(self): + logger = logging.getLogger('dim_aliases_test_logger') + records = [] + + class _Capture(logging.Handler): + def emit(self, record): + records.append(record) + + handler = _Capture(level=logging.WARNING) + logger.addHandler(handler) + try: + dim_aliases.enable(logger=logger, _stream=io.StringIO()) + self.assertEqual(len(records), 1) + self.assertEqual(records[0].levelno, logging.WARNING) + msg = records[0].getMessage() + self.assertIn('adjusted_vertical_layer_dimension_for_radiation', + msg) + self.assertIn('vertical_composition_dimension', msg) + self.assertIn('vertical_layer_dimension', msg) + finally: + logger.removeHandler(handler) + + +class TestCanonicalOn(_DimAliasesFixture): + """When the shim is enabled, the documented map applies.""" + + def setUp(self): + super().setUp() + dim_aliases.enable(_stream=io.StringIO()) + + def test_adjusted_radiation_collapses(self): + self.assertEqual( + dim_aliases.canonical( + 'adjusted_vertical_layer_dimension_for_radiation'), + 'vertical_layer_dimension', + ) + + def test_composition_collapses(self): + self.assertEqual( + dim_aliases.canonical('vertical_composition_dimension'), + 'vertical_layer_dimension', + ) + + def test_unknown_name_passes_through(self): + self.assertEqual( + dim_aliases.canonical('air_temperature'), 'air_temperature', + ) + + def test_canonical_target_is_idempotent(self): + # The representative itself is not in the alias map; calling + # canonical on it must be a no-op so repeated normalisation + # never drifts. + self.assertEqual( + dim_aliases.canonical('vertical_layer_dimension'), + 'vertical_layer_dimension', + ) + + +######################################################################## +# Integration through suite_resolver._canonical_dim +######################################################################## + +class TestCanonicalDimHook(_DimAliasesFixture): + """``_canonical_dim`` calls ``dim_aliases.canonical`` on the upper + bound only. Disabled mode keeps the original spelling; enabled + mode collapses every member of an alias group to one + representative so the per-position dim-identity check accepts the + pairing. + """ + + def test_disabled_keeps_aliased_dim_distinct(self): + a = _canonical_dim('adjusted_vertical_layer_dimension_for_radiation') + b = _canonical_dim('vertical_layer_dimension') + self.assertNotEqual(a, b) + + def test_enabled_collapses_radiation_alias(self): + dim_aliases.enable(_stream=io.StringIO()) + a = _canonical_dim('adjusted_vertical_layer_dimension_for_radiation') + b = _canonical_dim('vertical_layer_dimension') + self.assertEqual(a, b) + # And the collapsed form is the canonical representative. + self.assertEqual(a, 'ccpp_constant_one:vertical_layer_dimension') + + def test_enabled_collapses_composition_alias(self): + dim_aliases.enable(_stream=io.StringIO()) + a = _canonical_dim('vertical_composition_dimension') + b = _canonical_dim('vertical_layer_dimension') + self.assertEqual(a, b) + self.assertEqual(a, 'ccpp_constant_one:vertical_layer_dimension') + + def test_enabled_collapses_in_explicit_range_form(self): + # Aliasing applies on the *upper* bound of an explicit lower:upper + # form too. Lower bound never aliases. + dim_aliases.enable(_stream=io.StringIO()) + a = _canonical_dim( + 'ccpp_constant_one:vertical_composition_dimension') + b = _canonical_dim('vertical_layer_dimension') + self.assertEqual(a, b) + + def test_enabled_does_not_alias_unrelated_dim(self): + dim_aliases.enable(_stream=io.StringIO()) + a = _canonical_dim('horizontal_dimension') + b = _canonical_dim('vertical_layer_dimension') + self.assertNotEqual(a, b) + + def test_enabled_does_not_alias_lower_bound(self): + # Only the upper bound is rewritten. If somebody wrote + # 'adjusted_vertical_layer_dimension_for_radiation:foo' as a + # range, the lower bound stays verbatim — there is no host + # context in which an alias name appears as a loop-begin + # control var, so we keep this strict. + dim_aliases.enable(_stream=io.StringIO()) + a = _canonical_dim( + 'adjusted_vertical_layer_dimension_for_radiation:foo') + self.assertEqual( + a, + 'adjusted_vertical_layer_dimension_for_radiation:foo', + ) + + +######################################################################## +# Doctest loader for dim_aliases module +######################################################################## + +def load_tests(loader, tests, ignore): + tests.addTests(doctest.DocTestSuite(dim_aliases)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_host_cap.py b/unit-tests/test_host_cap.py new file mode 100644 index 00000000..b82848df --- /dev/null +++ b/unit-tests/test_host_cap.py @@ -0,0 +1,1379 @@ +"""Unit tests for generator.host_cap.""" + +import doctest +import os +import tempfile +import unittest +from unittest.mock import MagicMock + +from metadata.parse_tools import CCPPError +from generator.suite_resolver import resolve_suite +from generator.host_cap import ( + _all_ctrl_args_for_phase, + _arg_top_level_name, + _build_local_to_std_top_level_map, + _collect_host_io, + _emit_var_set_loop, + _generate_host_cap, + _suite_io_subroutine, + _suite_list_subroutine, + _suite_part_list_subroutine, + _suite_schemes_subroutine, + write_host_cap, +) +from test_suite_resolver import ( + _load_full_host_dict, + _load_scheme_store, + _parse_suite, +) + + +def _resolve(): + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + return resolve_suite(suite, store, hd) + + +def _generate(): + suite_resolution = _resolve() + return _generate_host_cap('test_host', ['test_simple'], [suite_resolution]) + + +class TestAllCtrlArgsForPhase(unittest.TestCase): + + def test_only_error_ctrl_args_in_test_case(self): + suite_resolution = _resolve() + # temp_calc_adjust uses errmsg/errflg which are now control vars. + args = _all_ctrl_args_for_phase([suite_resolution], 'run') + std_names = {a.standard_name for a in args} + self.assertEqual(std_names, {'ccpp_error_message', 'ccpp_error_code'}) + + def test_mismatched_lengths_raises(self): + from metadata.parse_tools import CCPPError + with self.assertRaises(CCPPError): + _generate_host_cap('test_host', ['a', 'b'], [_resolve()]) + + +class TestGenerateHostCapModule(unittest.TestCase): + """Static API: ccpp_register/init/final are mandatory entry points and + are always emitted with the minimal lifecycle signature.""" + + def setUp(self): + self.lines = _generate() + self.text = '\n'.join(self.lines) + + def test_module_header_comment(self): + self.assertTrue(self.lines[0].startswith('!')) + self.assertIn('test_host_ccpp_cap', self.lines[0]) + + def test_module_declaration(self): + self.assertIn('module test_host_ccpp_cap', self.text) + self.assertIn('end module test_host_ccpp_cap', self.text) + + def test_does_not_use_constituent_mod(self): + # Constituent merging is now opt-in via type=host (Task #6 follow-up). + self.assertNotIn('use ccpp_constituent_prop_mod', self.text) + self.assertNotIn('ccpp_model_constituents_t', self.text) + + def test_uses_suite_cap(self): + self.assertIn('use ccpp_test_simple_cap', self.text) + # Register is now mandatory and always imported. + self.assertIn('test_simple_register', self.text) + self.assertIn('test_simple_init', self.text) + self.assertIn('test_simple_final', self.text) + + def test_implicit_none_private(self): + self.assertIn('implicit none', self.text) + self.assertIn('private', self.text) + + def test_ccpp_register_public(self): + # ccpp_register is mandatory. + self.assertIn('public :: ccpp_register', self.text) + + def test_other_public_entry_points(self): + for ep in ('ccpp_init', 'ccpp_final', + 'ccpp_physics_init', 'ccpp_physics_timestep_init', + 'ccpp_physics_run', 'ccpp_physics_timestep_final', + 'ccpp_physics_final'): + self.assertIn('public :: {}'.format(ep), self.text) + + def test_contains_block(self): + self.assertIn('contains', self.lines) + + def test_no_constituent_reexport_when_absent(self): + # The test_simple fixture has no constituents — host_constituents + # module isn't emitted, so host_cap must not USE or re-export it. + self.assertNotIn('use ccpp_host_constituents', self.text) + self.assertNotIn('ccpp_register_constituents', self.text) + self.assertNotIn('ccpp_initialize_constituents', self.text) + + +class TestHostCapConstituentReexport(unittest.TestCase): + """When any suite uses constituent state, host_cap USEs + ccpp_host_constituents and re-publics every host-facing routine plus + the constituent object so hosts can ``use _ccpp_cap, only: ...`` + for everything they need from CCPP.""" + + def setUp(self): + from test_suite_resolver import ( + _load_constituent_host_dict, + _load_constituent_consumer_store, + _parse_suite, + ) + hd = _load_constituent_host_dict() + store = _load_constituent_consumer_store() + suite = _parse_suite('suite_consume_constituent.xml') + suite_resolution = resolve_suite(suite, store, hd) + self.text = '\n'.join( + _generate_host_cap('test_host', ['consume_consts'], [suite_resolution], host_dict=hd, + scheme_store=store), + ) + + def _expected_symbols(self): + return [ + 'ccpp_model_constituents_obj', + 'ccpp_register_constituents', + 'ccpp_initialize_constituents', + 'ccpp_is_scheme_constituent', + 'ccpp_number_constituents', + 'ccpp_gather_constituents', + 'ccpp_update_constituents', + 'ccpp_const_get_index', + 'ccpp_constituents_array', + 'ccpp_advected_constituents_array', + 'ccpp_model_const_properties', + 'ccpp_deallocate_dynamic_constituents', + ] + + def test_uses_host_constituents_module(self): + self.assertIn('use ccpp_host_constituents, only:', self.text) + + def test_all_symbols_imported(self): + for sym in self._expected_symbols(): + self.assertIn(sym, self.text, + 'symbol not imported: {}'.format(sym)) + + def test_all_symbols_re_public(self): + for sym in self._expected_symbols(): + self.assertIn('public :: {}'.format(sym), self.text) + + +class TestCcppRegisterMandatory(unittest.TestCase): + """ccpp_register is always emitted with the minimal lifecycle signature + and dispatches to every suite's _register routine.""" + + def setUp(self): + self.text = '\n'.join(_generate()) + + def test_subroutine_present(self): + self.assertIn('subroutine ccpp_register(', self.text) + self.assertIn('end subroutine ccpp_register', self.text) + + def test_signature_minimal_no_host_dict(self): + # With no host_dict, fallback local names are used. + self.assertIn( + 'subroutine ccpp_register(suite_name, errflg, errmsg)', self.text, + ) + + def test_no_constituents_arg(self): + # Constituents handling is opt-in; not in the signature any longer. + sig_block = self.text.split('subroutine ccpp_register')[1].split( + 'end subroutine ccpp_register' + )[0] + self.assertNotIn('constituents', sig_block) + + def test_dispatches_to_suite_register(self): + self.assertIn("case('test_simple')", self.text) + self.assertIn('call test_simple_register(errmsg, errflg)', self.text) + + def test_default_case_error(self): + self.assertIn("'ccpp_register: unknown suite:", self.text) + + +class TestCcppInitFinalSubroutines(unittest.TestCase): + + def setUp(self): + self.text = '\n'.join(_generate()) + + def test_init_subroutine(self): + self.assertIn( + 'subroutine ccpp_init(suite_name, errflg, errmsg)', self.text, + ) + self.assertIn('call test_simple_init(errmsg, errflg)', self.text) + self.assertIn("'ccpp_init: unknown suite:", self.text) + + def test_final_subroutine(self): + self.assertIn( + 'subroutine ccpp_final(suite_name, errflg, errmsg)', self.text, + ) + self.assertIn('call test_simple_final(errmsg, errflg)', self.text) + self.assertIn("'ccpp_final: unknown suite:", self.text) + + +class TestCcppPhysicsSubroutines(unittest.TestCase): + + def setUp(self): + self.text = '\n'.join(_generate()) + + def test_all_physics_subroutines_present(self): + for phase in ('init', 'timestep_init', 'run', 'timestep_final', 'final'): + self.assertIn('subroutine ccpp_physics_{}'.format(phase), self.text) + + def test_run_dispatches_to_suite_cap(self): + # No host_dict passed → no ctrl entries → no-arg call to suite cap. + self.assertIn('call test_simple_physics_run()', self.text) + + def test_init_dispatches_to_suite_cap(self): + self.assertIn('call test_simple_physics_init()', self.text) + + def test_final_dispatches_to_suite_cap(self): + self.assertIn('call test_simple_physics_final()', self.text) + + def test_physics_no_default_error_case_without_host_dict(self): + # When no host_dict is available the standard error-reporting + # control vars (ccpp_error_code / ccpp_error_message) aren't in + # scope, so the physics dispatch has nowhere to write a "unknown + # suite" message — case default is intentionally omitted. + run_block_start = self.text.index('subroutine ccpp_physics_run') + run_block_end = self.text.index('end subroutine ccpp_physics_run') + run_block = self.text[run_block_start:run_block_end] + self.assertNotIn('case default', run_block) + + def test_select_case_on_suite_name(self): + self.assertIn('select case(trim(suite_name))', self.text) + + +class TestCcppPhysicsUnknownSuiteErrors(unittest.TestCase): + """When the host provides ccpp_error_code / ccpp_error_message in + its control table, the physics dispatch ``select case`` MUST end + with a ``case default`` that sets errflg=1 and writes a message + naming the unknown suite — never silently fall through. + """ + + def setUp(self): + hd = _load_full_host_dict() + suite_resolution = _resolve() + self.text = '\n'.join(_generate_host_cap('test_host', ['test_simple'], [suite_resolution], hd)) + + def test_physics_run_has_default_case_with_errflg(self): + run_block_start = self.text.index('subroutine ccpp_physics_run') + run_block_end = self.text.index('end subroutine ccpp_physics_run') + run_block = self.text[run_block_start:run_block_end] + self.assertIn('case default', run_block) + # errflg must be set non-zero in the default branch. + self.assertRegex(run_block, r'case default[^!]*?errflg = 1') + + def test_physics_run_default_message_names_suite(self): + run_block_start = self.text.index('subroutine ccpp_physics_run') + run_block_end = self.text.index('end subroutine ccpp_physics_run') + run_block = self.text[run_block_start:run_block_end] + self.assertIn( + "ccpp_physics_run: unknown suite: ' // trim(suite_name)", + run_block, + ) + + +class TestMultipleSuites(unittest.TestCase): + """Static API with two suites uses select case for both.""" + + def setUp(self): + suite_resolution = _resolve() + from copy import deepcopy + sr2 = deepcopy(suite_resolution) + sr2.suite_name = 'suite_b' + lines = _generate_host_cap('test_host', ['test_simple', 'suite_b'], [suite_resolution, sr2]) + self.text = '\n'.join(lines) + + def test_both_suites_in_register(self): + # ccpp_register dispatches to all suites. + self.assertIn("case('test_simple')", self.text) + self.assertIn("case('suite_b')", self.text) + + def test_both_suite_caps_used(self): + self.assertIn('use ccpp_test_simple_cap', self.text) + self.assertIn('use ccpp_suite_b_cap', self.text) + + +class TestWriteHostCap(unittest.TestCase): + + def test_writes_file(self): + suite_resolution = _resolve() + with tempfile.TemporaryDirectory() as tmpdir: + path = write_host_cap('test_host', ['test_simple'], [suite_resolution], tmpdir) + self.assertTrue(os.path.isfile(path)) + self.assertEqual(os.path.basename(path), 'test_host_ccpp_cap.F90') + + def test_file_content(self): + suite_resolution = _resolve() + with tempfile.TemporaryDirectory() as tmpdir: + path = write_host_cap('test_host', ['test_simple'], [suite_resolution], tmpdir) + with open(path) as fh: + content = fh.read() + self.assertIn('module test_host_ccpp_cap', content) + # ccpp_register is now mandatory and always emitted. + self.assertIn('subroutine ccpp_register', content) + self.assertTrue(content.endswith('\n')) + + def test_creates_output_dir(self): + suite_resolution = _resolve() + with tempfile.TemporaryDirectory() as tmpdir: + subdir = os.path.join(tmpdir, 'api') + write_host_cap('test_host', ['test_simple'], [suite_resolution], subdir) + self.assertTrue(os.path.isdir(subdir)) + + def test_returns_absolute_path(self): + suite_resolution = _resolve() + with tempfile.TemporaryDirectory() as tmpdir: + path = write_host_cap('test_host', ['test_simple'], [suite_resolution], tmpdir) + self.assertTrue(os.path.isabs(path)) + + +class TestCcppInitMultiInstance(unittest.TestCase): + """ccpp_init includes instance_number when the host provides it; the new + minimal signature drops number_of_instances entirely.""" + + def setUp(self): + hd = _load_full_host_dict() + suite_resolution = _resolve() + lines = _generate_host_cap('test_host', ['test_simple'], [suite_resolution], hd) + self.text = '\n'.join(lines) + + def test_init_signature_has_instance_pair(self): + # host_full.meta declares the multi-instance pair (inst_num, + # ninstances). Both must appear in the lifecycle signature. + self.assertIn( + 'subroutine ccpp_init(suite_name, errflg, errmsg, inst_num, ninstances)', + self.text, + ) + + def test_init_signature_has_ninstances(self): + init_block = self.text.split('subroutine ccpp_init')[1].split( + 'end subroutine ccpp_init' + )[0] + self.assertIn('ninstances', init_block) + + def test_init_passes_inst_pair_to_suite(self): + self.assertIn( + 'call test_simple_init(inst_num, ninstances, errmsg, errflg)', + self.text, + ) + + def test_register_signature_has_instance_pair(self): + self.assertIn( + 'subroutine ccpp_register(suite_name, errflg, errmsg, inst_num, ninstances)', + self.text, + ) + + def test_final_signature_has_instance_pair(self): + # Final carries (inst_num, ninstances) for API symmetry with + # register/init even though the framework doesn't read + # ninstances at final time. + self.assertIn( + 'subroutine ccpp_final(suite_name, errflg, errmsg, inst_num, ninstances)', + self.text, + ) + + +class TestCcppInitSingleInstance(unittest.TestCase): + """ccpp_init drops inst_num when the host doesn't declare instance_number.""" + + def setUp(self): + hd = {k: v for k, v in _load_full_host_dict().items() + if k not in ('number_of_instances', 'instance_number')} + suite_resolution = _resolve() + lines = _generate_host_cap('test_host', ['test_simple'], [suite_resolution], hd) + self.text = '\n'.join(lines) + + def test_init_signature_no_instance_args(self): + self.assertIn( + 'subroutine ccpp_init(suite_name, errflg, errmsg)', self.text, + ) + init_block = self.text.split('subroutine ccpp_init')[1].split( + 'end subroutine ccpp_init' + )[0] + self.assertNotIn('ninstances', init_block) + self.assertNotIn('inst_num', init_block) + + def test_init_passes_no_extra_args_to_suite(self): + self.assertIn('call test_simple_init(errmsg, errflg)', self.text) + + +######################################################################## +# Suite-introspection: helpers +######################################################################## + +class TestEmitVarSetLoop(unittest.TestCase): + + def test_basic(self): + out = _emit_var_set_loop('x', ['a', 'b'], ' ') + self.assertEqual(out, [' allocate(x(2))', " x(1) = 'a'", " x(2) = 'b'"]) + + def test_empty(self): + out = _emit_var_set_loop('x', [], ' ') + self.assertEqual(out, [' allocate(x(0))']) + + def test_no_allocate(self): + out = _emit_var_set_loop('x', ['a'], ' ', allocate=False) + self.assertEqual(out, [" x(1) = 'a'"]) + + +class TestBuildLocalToStdTopLevelMap(unittest.TestCase): + """Reverse map covers top-level host_dict entries only (no DDT-leaf rows).""" + + def test_includes_top_level_entries(self): + # host_full has only plain leaves (no DDTs) → all entries top-level. + hd = _load_full_host_dict() + m = _build_local_to_std_top_level_map(hd) + self.assertEqual(m['gt0'], 'air_temperature') + self.assertEqual(m['ncols'], 'horizontal_dimension') + + def test_excludes_ddt_leaves(self): + from metadata.metadata_table import parse_metadata_file + from metadata.variable_resolver import build_flat_host_dict + hd = build_flat_host_dict( + parse_metadata_file(os.path.join( + os.path.dirname(__file__), 'sample_files', + 'host_with_ddt_instance.meta')), + [], + parse_metadata_file(os.path.join( + os.path.dirname(__file__), 'sample_files', + 'ddt_simple.meta')), + ) + m = _build_local_to_std_top_level_map(hd) + # The DDT instance itself appears (top-level). + self.assertEqual(m['gfs_statein'], 'gfs_statein') + # DDT-leaf local names ('phii', 'phil') are excluded. + self.assertNotIn('phii', m) + self.assertNotIn('phil', m) + + def test_none_returns_empty(self): + self.assertEqual(_build_local_to_std_top_level_map(None), {}) + + +class TestArgTopLevelName(unittest.TestCase): + """Collapse a flat DDT-leaf back to its top-level DDT instance name.""" + + def _make_arg(self, std_name, access_path): + """Mock ResolvedArg with just the fields _arg_top_level_name reads.""" + host_entry = MagicMock() + host_entry.access_path = access_path + arg = MagicMock() + arg.standard_name = std_name + arg.host_entry = host_entry + return arg + + def test_plain_leaf_unchanged(self): + arg = self._make_arg('air_temperature', 'gt0') + self.assertEqual(_arg_top_level_name(arg, {}), 'air_temperature') + + def test_ddt_leaf_collapsed(self): + arg = self._make_arg( + 'geopotential_at_interface', + 'gfs_statein(instance_number)%phii', + ) + m = {'gfs_statein': 'gfs_statein'} + self.assertEqual(_arg_top_level_name(arg, m), 'gfs_statein') + + def test_nested_ddt_collapses_to_outermost(self): + arg = self._make_arg('inner_field', 'outer(2)%inner%fld') + m = {'outer': 'outer_std_name'} + self.assertEqual(_arg_top_level_name(arg, m), 'outer_std_name') + + def test_unmapped_root_falls_back_to_arg_std_name(self): + # If the root local_name isn't in the map (inconsistent metadata), + # fall back to the arg's own standard_name. + arg = self._make_arg('foo', 'unknown(1)%bar') + self.assertEqual(_arg_top_level_name(arg, {}), 'foo') + + def test_no_host_entry(self): + arg = MagicMock() + arg.standard_name = 'baz' + arg.host_entry = None + self.assertEqual(_arg_top_level_name(arg, {'x': 'y'}), 'baz') + + +class TestCollectHostIo(unittest.TestCase): + """_collect_host_io: intent partitioning, control-var exclusion, sort.""" + + def setUp(self): + self.suite_resolution = _resolve() + self.hd = _load_full_host_dict() + + def test_includes_control_vars(self): + # temp_calc_adjust declares errflg/errmsg with intent=out — they + # appear in outputs. Matches original capgen's introspection. + inputs, outputs = _collect_host_io(self.suite_resolution, self.hd) + self.assertIn('ccpp_error_code', outputs) + self.assertIn('ccpp_error_message', outputs) + # …and not in inputs (intent=out only, not inout). + self.assertNotIn('ccpp_error_code', inputs) + self.assertNotIn('ccpp_error_message', inputs) + + def test_includes_host_args(self): + inputs, outputs = _collect_host_io(self.suite_resolution, self.hd) + # air_temperature is intent=inout in run phase → both lists. + self.assertIn('air_temperature', inputs) + self.assertIn('air_temperature', outputs) + + def test_sorted_outputs(self): + inputs, outputs = _collect_host_io(self.suite_resolution, self.hd) + self.assertEqual(inputs, sorted(inputs)) + self.assertEqual(outputs, sorted(outputs)) + + def test_collapse_ddts_no_ddts_unchanged(self): + # host_full has no DDTs → collapse is a no-op. + flat_in, flat_out = _collect_host_io(self.suite_resolution, self.hd, collapse_ddts=False) + coll_in, coll_out = _collect_host_io(self.suite_resolution, self.hd, collapse_ddts=True) + self.assertEqual(flat_in, coll_in) + self.assertEqual(flat_out, coll_out) + + def test_no_host_dict_collapse_falls_back(self): + # collapse_ddts=True without host_dict must not raise. + # With no DDTs the result is identical to the non-collapsed view. + no_hd_in, _ = _collect_host_io(self.suite_resolution, None, collapse_ddts=True) + flat_in, _ = _collect_host_io(self.suite_resolution, self.hd, collapse_ddts=False) + self.assertEqual(no_hd_in, flat_in) + + +class TestCollectHostIoIncludesNonHostSources(unittest.TestCase): + """_collect_host_io includes constituent args + register-phase + ccpp_constituent_properties_t args + control vars in the introspection + lists (matches original capgen). Only suite-owned vars are excluded.""" + + def setUp(self): + from test_suite_resolver import ( + _load_constituent_host_dict, + _load_constituent_consumer_store, + _load_constituent_scheme_store, + _parse_suite, + ) + # Mix: a register-phase producer scheme + a consumer scheme. + # Build a SuiteResolution for each, pass both to _collect_host_io. + self.hd = _load_constituent_host_dict() + consumer_store = _load_constituent_consumer_store() + register_store = _load_constituent_scheme_store() + consumer_suite = _parse_suite('suite_consume_constituent.xml') + register_suite = _parse_suite('suite_register_constituents.xml') + self.consumer_sr = resolve_suite(consumer_suite, consumer_store, self.hd) + self.register_sr = resolve_suite(register_suite, register_store, self.hd) + + def test_consumer_base_constituent_in_inputs(self): + inputs, _ = _collect_host_io(self.consumer_sr, self.hd) + # cldliq is intent=in advected=true → in inputs. + self.assertIn('cloud_liquid_water_mixing_ratio', inputs) + + def test_consumer_tendency_in_outputs(self): + _, outputs = _collect_host_io(self.consumer_sr, self.hd) + # tend_cldliq is intent=out constituent=true → in outputs. + self.assertIn( + 'tendency_of_cloud_liquid_water_mixing_ratio', outputs, + ) + + def test_register_phase_properties_t_in_outputs(self): + # The register-phase scheme declares dyn_const as intent=out + # ccpp_constituent_properties_t — appears in the output list. + _, outputs = _collect_host_io(self.register_sr, self.hd) + self.assertIn('dynamic_constituents_for_register_test', outputs) + + def test_control_vars_in_outputs(self): + # The register scheme also declares errmsg/errflg with intent=out. + _, outputs = _collect_host_io(self.register_sr, self.hd) + self.assertIn('ccpp_error_code', outputs) + self.assertIn('ccpp_error_message', outputs) + + +class TestCollectHostIoIncludesFrameworkDims(unittest.TestCase): + """``number_of_ccpp_constituents`` (and any other framework-constituent + dim that appears only as a dim token in scheme metadata) is included + in the introspection inputs list — matches original capgen. Host-side + dims (horizontal_dimension, vertical_layer_dimension) are NOT + included; they're stable host structure.""" + + def setUp(self): + from test_suite_resolver import ( + _load_full_host_dict, _load_scheme_store, _parse_suite, + ) + # Use the consume_constituent fixture — the scheme references + # number_of_ccpp_constituents as a dim of ccpp_constituents. + # But that fixture only declares 2D constituent vars, not the + # 3D ccpp_constituents directly. Build a minimal fixture + # specifically for this test. + from metadata.metadata_table import _parse_lines + from metadata.variable_resolver import SchemeStore + scheme_text = ( + '[ccpp-table-properties]\n' + ' name = uses_const_array\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = uses_const_array_run\n' + ' type = scheme\n' + '[ const ]\n' + ' standard_name = ccpp_constituents\n' + ' units = none\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension, ' + 'number_of_ccpp_constituents)\n' + ' type = real | kind = kind_phys\n' + ' intent = inout\n' + ) + tables = _parse_lines(scheme_text.splitlines(keepends=True), 't.meta') + store = SchemeStore.build_from(tables) + + # Use the host-only dict (no ccpp_model_constituents_t DDT + # instance) so the host-wins rule doesn't fire and the scheme's + # ccpp_constituents arg routes through capgen's + # auto-provisioning path — that's the code path that surfaces + # number_of_ccpp_constituents as an input via used_const_dim_std_names. + self.hd = _load_full_host_dict() + # Parse a one-scheme suite XML inline. + import tempfile, os, logging + from generator.suite_xml import parse_suite_xml + suite_xml = ( + '\n' + '\n' + ' uses_const_array\n' + '\n' + ) + with tempfile.TemporaryDirectory() as tmp: + xml_path = os.path.join(tmp, 'suite.xml') + with open(xml_path, 'w') as fh: + fh.write(suite_xml) + suite = parse_suite_xml(xml_path, tmp, logging.getLogger('t'), + skip_validation=True) + self.suite_resolution = resolve_suite(suite, store, self.hd) + + def test_number_of_ccpp_constituents_in_inputs(self): + inputs, _ = _collect_host_io(self.suite_resolution, self.hd) + self.assertIn('number_of_ccpp_constituents', inputs) + + def test_horizontal_dim_not_in_inputs(self): + inputs, _ = _collect_host_io(self.suite_resolution, self.hd) + # Sanity: the host-side dims are NOT included even though they + # appear as scheme arg dimensions. + self.assertNotIn('horizontal_dimension', inputs) + self.assertNotIn('vertical_layer_dimension', inputs) + + +class TestCollectHostIoIncludesSubcycleLoopBound(unittest.TestCase): + """A subcycle ``loop=""`` bound is supplied by the host + (it controls the per-cap do-loop count) and must therefore appear + in the introspection inputs list — otherwise a host comparing its + declared variables against ``ccpp_physics_suite_variables`` will + silently miss the dependency.""" + + def setUp(self): + from test_suite_resolver import ( + _load_full_host_dict, _load_scheme_store, _parse_suite, + ) + from metadata.metadata_table import _parse_lines + from metadata.variable_resolver import build_flat_host_dict + from generator.suite_resolver import resolve_suite + + # Pair the standard host_full + scheme_multipart with a small + # extension that declares the subcycle-count std name. + helper_src = ''' +[ccpp-table-properties] + name = subcycle_helper + type = host +[ccpp-arg-table] + name = subcycle_helper + type = host +[ n_sub ] + standard_name = num_subcycles_for_my_scheme + units = count + dimensions = () + type = integer +''' + helper_tbls = _parse_lines( + helper_src.splitlines(keepends=True), 'h.meta', + ) + + from test_suite_resolver import _SAMPLES_DIR + from metadata.metadata_table import parse_metadata_file + host_tbls = parse_metadata_file( + os.path.join(_SAMPLES_DIR, 'host_full.meta') + ) + ctrl_tbls = parse_metadata_file( + os.path.join(_SAMPLES_DIR, 'control_full.meta') + ) + host_only = [t for t in host_tbls if t.table_type == 'host'] + ctrl_only = [t for t in ctrl_tbls if t.table_type == 'control'] + ddt_only = [t for t in host_tbls if t.table_type == 'ddt'] + self.hd = build_flat_host_dict(host_only + helper_tbls, ctrl_only, ddt_only) + + scheme_store = _load_scheme_store() + + suite_xml = ( + '\n' + '\n' + ' \n' + ' \n' + ' temp_calc_adjust\n' + ' \n' + ' \n' + '\n' + ) + import tempfile, logging + from generator.suite_xml import parse_suite_xml + with tempfile.TemporaryDirectory() as tmp: + xml_path = os.path.join(tmp, 's.xml') + with open(xml_path, 'w') as fh: + fh.write(suite_xml) + suite = parse_suite_xml(xml_path, tmp, logging.getLogger('t'), + skip_validation=True) + self.suite_resolution = resolve_suite(suite, scheme_store, self.hd) + + def test_subcycle_std_name_in_inputs(self): + inputs, _outputs = _collect_host_io(self.suite_resolution, self.hd) + self.assertIn('num_subcycles_for_my_scheme', inputs) + + def test_integer_literal_subcycle_does_not_pollute(self): + """A literal-integer subcycle bound (``loop="3"``) has no + ``loop_std_name`` set and therefore contributes nothing to + the inputs list.""" + from test_suite_resolver import ( + _load_scheme_store, _load_full_host_dict, + ) + from generator.suite_resolver import resolve_suite + import tempfile, logging + from generator.suite_xml import parse_suite_xml + + hd = _load_full_host_dict() + store = _load_scheme_store() + suite_xml = ( + '\n' + '\n' + ' \n' + ' \n' + ' temp_calc_adjust\n' + ' \n' + ' \n' + '\n' + ) + with tempfile.TemporaryDirectory() as tmp: + xml_path = os.path.join(tmp, 's.xml') + with open(xml_path, 'w') as fh: + fh.write(suite_xml) + suite = parse_suite_xml(xml_path, tmp, logging.getLogger('t'), + skip_validation=True) + suite_resolution = resolve_suite(suite, store, hd) + inputs, _ = _collect_host_io(suite_resolution, hd) + # No spurious integer / std-name additions from the literal loop. + self.assertNotIn('3', inputs) + + +class TestCollectHostIoIncludesActiveExpr(unittest.TestCase): + """A flag referenced via ``active=()`` on a host variable + must appear in the introspection inputs list — even if no scheme + declares it as a direct argument. The host needs to know the + suite consumes the flag to decide whether the optional variables + are present.""" + + def setUp(self): + from metadata.metadata_table import _parse_lines + from metadata.variable_resolver import build_flat_host_dict, SchemeStore + from generator.suite_resolver import resolve_suite + import tempfile, logging + from generator.suite_xml import parse_suite_xml + + # Host has a flag (``flag_for_passive_check``) referenced only as + # an active=() expression on another host var. No scheme takes + # the flag as a direct argument — without the active-expr walk + # in _collect_host_io it would silently disappear. + # + # The matching scheme arg is declared optional, which the + # resolver's active+optional coherence check requires (host + # ``active`` means the host's variable is only valid when the + # condition holds; the cap honors that via the optional/ + # pointer-association pattern). + host_src = ''' +[ccpp-table-properties] + name = active_helper + type = host +[ccpp-arg-table] + name = active_helper + type = host +[ ncols ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + units = count + dimensions = () + type = integer +[ flag_passive ] + standard_name = flag_for_passive_check + units = flag + dimensions = () + type = logical +[ dt ] + standard_name = time_step_for_physics + units = s + dimensions = () + type = real + kind = kind_phys +[ gt0 ] + standard_name = air_temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + active = (flag_for_passive_check) +''' + # Inline scheme whose ``temp`` arg is optional — paired with the + # host-side active above. + scheme_src = ''' +[ccpp-table-properties] + name = active_scheme + type = scheme +[ccpp-arg-table] + name = active_scheme_run + type = scheme +[ im ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + intent = in +[ temp ] + standard_name = air_temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout + optional = True +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + from test_suite_resolver import _SAMPLES_DIR + from metadata.metadata_table import parse_metadata_file + ctrl_tbls = parse_metadata_file( + os.path.join(_SAMPLES_DIR, 'control_full.meta') + ) + host_tbls = _parse_lines(host_src.splitlines(keepends=True), 'h.meta') + ctrl_only = [t for t in ctrl_tbls if t.table_type == 'control'] + self.hd = build_flat_host_dict(host_tbls, ctrl_only, []) + + scheme_tbls = _parse_lines( + scheme_src.splitlines(keepends=True), 's.meta', + ) + store = SchemeStore.build_from(scheme_tbls) + suite_xml = ( + '\n' + '\n' + ' active_scheme\n' + '\n' + ) + with tempfile.TemporaryDirectory() as tmp: + xml_path = os.path.join(tmp, 's.xml') + with open(xml_path, 'w') as fh: + fh.write(suite_xml) + suite = parse_suite_xml(xml_path, tmp, logging.getLogger('t'), + skip_validation=True) + self.suite_resolution = resolve_suite(suite, store, self.hd) + + def test_active_flag_in_inputs(self): + inputs, _outputs = _collect_host_io(self.suite_resolution, self.hd) + self.assertIn('flag_for_passive_check', inputs) + + def test_active_flag_not_in_outputs(self): + """The flag is a pure input — it must not leak into outputs.""" + _, outputs = _collect_host_io(self.suite_resolution, self.hd) + self.assertNotIn('flag_for_passive_check', outputs) + + +######################################################################## +# Suite-introspection: ccpp_physics_suite_list +######################################################################## + +class TestSuiteListSubroutine(unittest.TestCase): + + def test_single_suite(self): + text = '\n'.join(_suite_list_subroutine(['test_simple'])) + self.assertIn('subroutine ccpp_physics_suite_list(suites)', text) + self.assertIn( + 'character(len=*), allocatable, intent(out) :: suites(:)', text, + ) + self.assertIn('allocate(suites(1))', text) + self.assertIn("suites(1) = 'test_simple'", text) + self.assertIn('end subroutine ccpp_physics_suite_list', text) + + def test_multi_suite(self): + text = '\n'.join(_suite_list_subroutine(['a', 'b', 'c'])) + self.assertIn('allocate(suites(3))', text) + self.assertIn("suites(1) = 'a'", text) + self.assertIn("suites(2) = 'b'", text) + self.assertIn("suites(3) = 'c'", text) + + +######################################################################## +# Suite-introspection: ccpp_physics_suite_part_list +######################################################################## + +class TestSuitePartListSubroutine(unittest.TestCase): + + def setUp(self): + self.suite_resolution = _resolve() + self.text = '\n'.join(_suite_part_list_subroutine(['test_simple'], [self.suite_resolution])) + + def test_signature(self): + self.assertIn('subroutine ccpp_physics_suite_part_list(', self.text) + self.assertIn( + 'character(len=*), intent(in) :: suite_name', self.text, + ) + self.assertIn( + 'character(len=*), allocatable, intent(out) :: part_list(:)', self.text, + ) + self.assertIn('integer, intent(out) :: errflg', self.text) + + def test_dispatch_and_groups(self): + self.assertIn("case ('test_simple')", self.text) + # test_simple has one group named 'physics'. + group_names = [g.group_name for g in self.suite_resolution.groups] + for i, gname in enumerate(group_names): + self.assertIn("part_list({}) = '{}'".format(i + 1, gname), self.text) + self.assertIn('allocate(part_list({}))'.format(len(group_names)), self.text) + + def test_default_error_case(self): + self.assertIn('case default', self.text) + self.assertIn('errflg = 1', self.text) + self.assertIn( + "errmsg = 'ccpp_physics_suite_part_list: unknown suite: '", + self.text, + ) + + def test_initialises_outputs(self): + # errmsg/errflg cleared at top of routine. + self.assertIn("errmsg = ''", self.text) + self.assertIn('errflg = 0', self.text) + + def test_errmsg_is_assumed_length(self): + """The errmsg dummy must be ``character(len=*)`` so the host can + pass any character length without copy-in/copy-out (e.g. the + host might use ``character(len=256)`` while the framework's + scheme metadata uses ``len=512``).""" + self.assertIn('character(len=*), intent(out) :: errmsg', + self.text) + self.assertNotIn('character(len=512)', self.text) + self.assertNotIn('character(len=256)', self.text) + + +######################################################################## +# Suite-introspection: ccpp_physics_suite_schemes +######################################################################## + +class TestSuiteSchemesSubroutine(unittest.TestCase): + + def setUp(self): + self.suite_resolution = _resolve() + self.text = '\n'.join( + _suite_schemes_subroutine(['test_simple'], [self.suite_resolution]) + ) + + def test_signature(self): + self.assertIn('subroutine ccpp_physics_suite_schemes(', self.text) + self.assertIn( + 'character(len=*), allocatable, intent(out) :: scheme_list(:)', + self.text, + ) + + def test_lists_temp_calc_adjust(self): + # test_simple's only scheme (across all phases) is temp_calc_adjust. + self.assertIn("scheme_list(1) = 'temp_calc_adjust'", self.text) + self.assertIn('allocate(scheme_list(1))', self.text) + + def test_dedup_across_phases(self): + # temp_calc_adjust appears in init, run, and final phases — must + # appear exactly once in the emitted list. + n = self.text.count("scheme_list(1) = 'temp_calc_adjust'") + self.assertEqual(n, 1) + self.assertNotIn("scheme_list(2) = 'temp_calc_adjust'", self.text) + + def test_default_error_case(self): + self.assertIn( + "errmsg = 'ccpp_physics_suite_schemes: unknown suite: '", + self.text, + ) + + def test_errmsg_is_assumed_length(self): + self.assertIn('character(len=*), intent(out) :: errmsg', + self.text) + self.assertNotIn('character(len=512)', self.text) + self.assertNotIn('character(len=256)', self.text) + + +######################################################################## +# Suite-introspection: ccpp_physics_suite_variables +######################################################################## + +class TestSuiteVariablesSubroutine(unittest.TestCase): + + def setUp(self): + self.suite_resolution = _resolve() + self.hd = _load_full_host_dict() + self.text = '\n'.join(_suite_io_subroutine( + ['test_simple'], [self.suite_resolution], self.hd, collapse_ddts=False, + )) + + def test_subroutine_name(self): + self.assertIn('subroutine ccpp_physics_suite_variables(', self.text) + self.assertIn('end subroutine ccpp_physics_suite_variables', self.text) + + def test_signature_includes_optional_filters(self): + self.assertIn( + 'logical, optional, intent(in) :: input_vars', + self.text, + ) + self.assertIn( + 'logical, optional, intent(in) :: output_vars', + self.text, + ) + + def test_no_struct_elements_arg(self): + # struct_elements is intentionally dropped (was a no-op in capgen). + self.assertNotIn('struct_elements', self.text) + + def test_errmsg_is_assumed_length(self): + """``suite_variables`` (and ``suite_host_data`` — they share the + same emitter) must declare errmsg as ``character(len=*)`` so the + host can pass any character length.""" + self.assertIn('character(len=*), intent(out) :: errmsg', + self.text) + self.assertNotIn('character(len=512)', self.text) + self.assertNotIn('character(len=256)', self.text) + + def test_three_branch_dispatch(self): + self.assertIn('if (input_vars_use .and. output_vars_use) then', self.text) + self.assertIn('else if (input_vars_use) then', self.text) + self.assertIn('else if (output_vars_use) then', self.text) + # Empty fall-through branch. + self.assertIn('allocate(variable_list(0))', self.text) + + def test_includes_control_vars(self): + # ccpp_error_code and ccpp_error_message DO appear in the emitted + # variable list literals — they're scheme args (intent=out) and + # are part of the host-facing surface (matches original capgen). + self.assertIn("'ccpp_error_code'", self.text) + self.assertIn("'ccpp_error_message'", self.text) + + def test_includes_host_data_vars(self): + # air_temperature is in the host metadata and is intent=inout in run. + self.assertIn("'air_temperature'", self.text) + + def test_default_present_check(self): + self.assertIn('if (present(input_vars)) then', self.text) + self.assertIn('if (present(output_vars)) then', self.text) + self.assertIn('input_vars_use = .true.', self.text) + self.assertIn('output_vars_use = .true.', self.text) + + def test_default_error_case(self): + self.assertIn( + "errmsg = 'ccpp_physics_suite_variables: unknown suite: '", + self.text, + ) + + +######################################################################## +# Suite-introspection: ccpp_physics_suite_host_data +######################################################################## + +class TestSuiteHostDataSubroutine(unittest.TestCase): + """Same shape as _variables; differs only in DDT collapsing.""" + + def setUp(self): + self.suite_resolution = _resolve() + self.hd = _load_full_host_dict() + self.text = '\n'.join(_suite_io_subroutine( + ['test_simple'], [self.suite_resolution], self.hd, collapse_ddts=True, + )) + + def test_subroutine_name(self): + self.assertIn('subroutine ccpp_physics_suite_host_data(', self.text) + self.assertIn('end subroutine ccpp_physics_suite_host_data', self.text) + + def test_default_error_case(self): + self.assertIn( + "errmsg = 'ccpp_physics_suite_host_data: unknown suite: '", + self.text, + ) + + def test_no_ddts_matches_variables(self): + # host_full has no DDTs → the routine bodies (modulo the subroutine + # name and error message) should contain the same variable + # literals as ..._variables. + var_text = '\n'.join(_suite_io_subroutine( + ['test_simple'], [self.suite_resolution], self.hd, collapse_ddts=False, + )) + # A spot-check: any host-data variable in one is in the other. + self.assertIn("'air_temperature'", self.text) + self.assertIn("'air_temperature'", var_text) + + +######################################################################## +# Suite-introspection: full module wiring +######################################################################## + +class TestIntrospectionRoutinesInModule(unittest.TestCase): + """Verify all five introspection routines are emitted and made public.""" + + def setUp(self): + self.text = '\n'.join(_generate()) + + def test_all_routines_present(self): + for sub in ( + 'ccpp_physics_suite_list', + 'ccpp_physics_suite_part_list', + 'ccpp_physics_suite_schemes', + 'ccpp_physics_suite_variables', + 'ccpp_physics_suite_host_data', + ): + self.assertIn('subroutine {}'.format(sub), self.text) + self.assertIn('end subroutine {}'.format(sub), self.text) + self.assertIn('public :: {}'.format(sub), self.text) + + +######################################################################## +# Suite-introspection: --no-host-introspection stub bodies +######################################################################## + +class TestNoHostIntrospectionStubBodies(unittest.TestCase): + """When ``--no-host-introspection`` is set, each of the five + introspection routines retains its signature but the body is + replaced with an errflg=1 stub (or, for suite_list, an error_unit + write + empty allocation). Tests assert the stub shape per routine + and that signatures remain stable so existing host callers still + link.""" + + _DISABLED_MSG = ( + 'suite introspection disabled at code-generation time; ' + 'regenerate caps without --no-host-introspection' + ) + + @classmethod + def setUpClass(cls): + cls.suite_resolution = _resolve() + cls.hd = _load_full_host_dict() + + def test_suite_list_writes_error_unit_and_allocates_empty(self): + text = '\n'.join(_suite_list_subroutine( + ['a', 'b', 'c'], stub_body=True, + )) + # Signature preserved. + self.assertIn('subroutine ccpp_physics_suite_list(suites)', text) + # Stub body: error_unit message, empty allocation. + self.assertIn('write(error_unit,', text) + self.assertIn('ccpp_physics_suite_list:', text) + self.assertIn(self._DISABLED_MSG, text) + self.assertIn('allocate(suites(0))', text) + # The functional body must NOT appear. + self.assertNotIn("suites(1) = 'a'", text) + self.assertNotIn('allocate(suites(3))', text) + + def test_suite_part_list_stub(self): + text = '\n'.join(_suite_part_list_subroutine( + ['test_simple'], [self.suite_resolution], stub_body=True, + )) + self.assertIn('subroutine ccpp_physics_suite_part_list(', text) + self.assertIn('errflg = 1', text) + self.assertIn('ccpp_physics_suite_part_list: ' + self._DISABLED_MSG, + text) + self.assertIn('allocate(part_list(0))', text) + # Functional dispatch must not appear. + self.assertNotIn("case ('test_simple')", text) + self.assertNotIn('select case', text) + + def test_suite_schemes_stub(self): + text = '\n'.join(_suite_schemes_subroutine( + ['test_simple'], [self.suite_resolution], stub_body=True, + )) + self.assertIn('subroutine ccpp_physics_suite_schemes(', text) + self.assertIn('errflg = 1', text) + self.assertIn('ccpp_physics_suite_schemes: ' + self._DISABLED_MSG, + text) + self.assertIn('allocate(scheme_list(0))', text) + self.assertNotIn("scheme_list(1) =", text) + + def test_suite_variables_stub(self): + text = '\n'.join(_suite_io_subroutine( + ['test_simple'], [self.suite_resolution], self.hd, + collapse_ddts=False, stub_body=True, + )) + self.assertIn('subroutine ccpp_physics_suite_variables(', text) + self.assertIn('errflg = 1', text) + self.assertIn('ccpp_physics_suite_variables: ' + self._DISABLED_MSG, + text) + self.assertIn('allocate(variable_list(0))', text) + # The huge case-block must NOT appear. + self.assertNotIn('select case (trim(suite_name))', text) + self.assertNotIn("case ('test_simple')", text) + # Optional dummies are still declared so existing callers still + # type-check. + self.assertIn('logical, optional, intent(in) :: input_vars', + text) + self.assertIn('logical, optional, intent(in) :: output_vars', + text) + + def test_suite_host_data_stub(self): + text = '\n'.join(_suite_io_subroutine( + ['test_simple'], [self.suite_resolution], self.hd, + collapse_ddts=True, stub_body=True, + )) + self.assertIn('subroutine ccpp_physics_suite_host_data(', text) + self.assertIn('errflg = 1', text) + self.assertIn('ccpp_physics_suite_host_data: ' + self._DISABLED_MSG, + text) + self.assertIn('allocate(variable_list(0))', text) + self.assertNotIn('select case (trim(suite_name))', text) + + def test_module_imports_error_unit_unconditionally(self): + # error_unit is always imported because every cap subroutine + # emits a gated ``if (trace) write(error_unit, *) ...`` line. + # Stub-on and stub-off both include the same USE. + for stub in (True, False): + text = '\n'.join(_generate_host_cap( + 'test_host', + ['test_simple'], [self.suite_resolution], self.hd, + no_host_introspection=stub, + )) + self.assertIn( + 'use, intrinsic :: iso_fortran_env, only: error_unit', + text, + msg='no_host_introspection={}'.format(stub), + ) + + def test_public_declarations_unchanged_when_stubbed(self): + # All five introspection routines remain public — callers must + # still link against them. + text = '\n'.join(_generate_host_cap( + 'test_host', + ['test_simple'], [self.suite_resolution], self.hd, + no_host_introspection=True, + )) + for sub in ( + 'ccpp_physics_suite_list', + 'ccpp_physics_suite_part_list', + 'ccpp_physics_suite_schemes', + 'ccpp_physics_suite_variables', + 'ccpp_physics_suite_host_data', + ): + self.assertIn('public :: {}'.format(sub), text) + self.assertIn('subroutine {}'.format(sub), text) + self.assertIn('end subroutine {}'.format(sub), text) + + def test_line_count_drops_dramatically(self): + """The motivating case: 33k+ lines → ~800. We don't have 80 + suites in unit-test fixtures, but even with one suite the + stubbed module must be strictly shorter than the full one.""" + full = _generate_host_cap('test_host', ['test_simple'], [self.suite_resolution], + self.hd, no_host_introspection=False) + stub = _generate_host_cap('test_host', ['test_simple'], [self.suite_resolution], + self.hd, no_host_introspection=True) + self.assertLess(len(stub), len(full), + 'stubbed module should be shorter than the full one ' + '(full={}, stub={})'.format(len(full), len(stub))) + + def test_write_host_cap_passes_flag_through(self): + """``write_host_cap(no_host_introspection=True, ...)`` must + produce a file containing stub bodies, not full ones.""" + with tempfile.TemporaryDirectory() as tmpdir: + out_path = write_host_cap( + 'test_host', + ['test_simple'], [self.suite_resolution], tmpdir, self.hd, + no_host_introspection=True, + ) + with open(out_path) as fh: + text = fh.read() + self.assertIn('ccpp_physics_suite_variables: ' + self._DISABLED_MSG, + text) + self.assertIn( + 'use, intrinsic :: iso_fortran_env, only: error_unit', text, + ) + + +class TestTraceEmission(unittest.TestCase): + """The generated static API always carries a module-level ``trace`` + parameter (default .false.) and a gated ``write(error_unit,*)`` in + every cap subroutine that has at least one intent(in) control dummy. + ``--trace`` flips the parameter default to .true. + """ + + def setUp(self): + self.hd = _load_full_host_dict() + self.suite_resolution = _resolve() + + def test_module_gate_default_off(self): + text = '\n'.join(_generate_host_cap( + 'test_host', + ['test_simple'], [self.suite_resolution], self.hd, + )) + self.assertIn('logical, parameter :: trace = .false.', text) + self.assertNotIn('logical, parameter :: trace = .true.', text) + + def test_module_gate_default_on(self): + text = '\n'.join(_generate_host_cap( + 'test_host', + ['test_simple'], [self.suite_resolution], self.hd, + trace=True, + )) + self.assertIn('logical, parameter :: trace = .true.', text) + self.assertNotIn('logical, parameter :: trace = .false.', text) + + def test_trace_block_present_in_physics_phases(self): + text = '\n'.join(_generate_host_cap( + 'test_host', + ['test_simple'], [self.suite_resolution], self.hd, + )) + # Every ccpp_physics_ dispatch has a gated write. + for phase in ('init', 'timestep_init', 'run', + 'timestep_final', 'final'): + self.assertIn( + "'CCPP TRACE ccpp_physics_{}:'".format(phase), + text, + msg='trace string missing for phase {}'.format(phase), + ) + + def test_trace_block_present_in_lifecycle_routines(self): + text = '\n'.join(_generate_host_cap( + 'test_host', + ['test_simple'], [self.suite_resolution], self.hd, + )) + for sub in ('ccpp_register', 'ccpp_init', 'ccpp_final'): + self.assertIn( + "'CCPP TRACE {}:'".format(sub), text, + msg='trace string missing for {}'.format(sub), + ) + + def test_write_host_cap_threads_trace_flag(self): + with tempfile.TemporaryDirectory() as tmpdir: + out_path = write_host_cap( + 'test_host', + ['test_simple'], [self.suite_resolution], tmpdir, self.hd, + trace=True, + ) + with open(out_path) as fh: + text = fh.read() + self.assertIn('logical, parameter :: trace = .true.', text) + + +def load_tests(loader, tests, ignore): + import generator.host_cap as sa + tests.addTests(doctest.DocTestSuite(sa)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_host_constituents.py b/unit-tests/test_host_constituents.py new file mode 100644 index 00000000..0511c692 --- /dev/null +++ b/unit-tests/test_host_constituents.py @@ -0,0 +1,744 @@ +"""Unit tests for ``generator.host_constituents``.""" + +import doctest +import os +import unittest + +from generator.suite_resolver import resolve_suite +from generator.host_constituents import ( + _any_constituent_state, + _all_index_names, + _suites_with_register_consts, + _generate_host_constituents, +) + + +def _resolve_consumer(): + """Resolve the consume_constituent fixture; return (suite_resolution, host_dict).""" + from test_suite_resolver import ( + _load_constituent_host_dict, + _load_constituent_consumer_store, + _parse_suite, + ) + hd = _load_constituent_host_dict() + store = _load_constituent_consumer_store() + suite = _parse_suite('suite_consume_constituent.xml') + return resolve_suite(suite, store, hd), hd + + +def _resolve_register(): + """Resolve the register_constituents fixture; return (suite_resolution, host_dict).""" + from test_suite_resolver import ( + _load_constituent_host_dict, + _load_constituent_scheme_store, + _parse_suite, + ) + hd = _load_constituent_host_dict() + store = _load_constituent_scheme_store() + suite = _parse_suite('suite_register_constituents.xml') + return resolve_suite(suite, store, hd), hd + + +def _resolve_simple(): + """Resolve the no-constituent fixture; return (suite_resolution, host_dict).""" + from test_suite_resolver import ( + _load_full_host_dict, + _load_scheme_store, + _parse_suite, + ) + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + return resolve_suite(suite, store, hd), hd + + +def _render_consumer(): + suite_resolution, hd = _resolve_consumer() + return '\n'.join(_generate_host_constituents([suite_resolution], host_dict=hd)) + + +def _render_register(): + suite_resolution, hd = _resolve_register() + return '\n'.join(_generate_host_constituents([suite_resolution], host_dict=hd)) + + +class TestAggregationHelpers(unittest.TestCase): + """``_any_constituent_state`` / ``_all_index_names`` / ``_suites_with_register_consts``.""" + + def test_any_when_consumer_only(self): + suite_resolution, _hd = _resolve_consumer() + self.assertTrue(_any_constituent_state([suite_resolution])) + + def test_any_when_register_only(self): + suite_resolution, _hd = _resolve_register() + self.assertTrue(_any_constituent_state([suite_resolution])) + + def test_any_when_neither(self): + suite_resolution, _hd = _resolve_simple() + self.assertFalse(_any_constituent_state([suite_resolution])) + + def test_index_names_aggregated(self): + consumer, _ch = _resolve_consumer() + register, _rh = _resolve_register() + names = _all_index_names([consumer, register]) + # Only the consumer side names a constituent in this fixture + # (register-phase scheme produces dyn_const3, but that std name + # is never read back via index_of_). + self.assertEqual(names, ['cloud_liquid_water_mixing_ratio']) + + def test_register_suites_listed(self): + consumer, _ch = _resolve_consumer() + register, _rh = _resolve_register() + self.assertEqual( + _suites_with_register_consts([consumer, register]), + ['reg_consts'], + ) + + def test_register_suites_include_auto_cloned_only(self): + # auto-clone-constituents: the legacy shim populates + # ``SuiteResolution.auto_cloned_constituents`` but leaves + # ``constituent_register_calls`` empty (no real register + # scheme exists). host_constituents.F90 still has to declare + # the per-suite ``_dynamic_constituents`` buffer + # because the suite cap emits a USE on it. Regression for + # CAM-SIMA kessler_test build (2026-06-03). + from generator.suite_resolver import SuiteResolution, AutoCloneEntry + sr = SuiteResolution( + suite_name='kessler_test', + auto_cloned_constituents=[AutoCloneEntry( + std_name='water_vapor', long_name='', diag_name='qv', + units='kg kg-1', vertical_dim='vertical_layer_dimension', + advected=True, molar_mass=0.0, default_value=None, + min_value=None, water_species=None, mixing_ratio_type=None, + )], + ) + self.assertEqual( + _suites_with_register_consts([sr]), + ['kessler_test'], + ) + + def test_register_suites_excludes_pure_consumer(self): + # auto-clone-constituents: no register calls AND no auto-cloned + # entries -> suite stays off the list so the buffer is not + # declared. Companion to test_register_suites_include_auto_cloned_only; + # delete together when the shim retires. + from generator.suite_resolver import SuiteResolution + sr = SuiteResolution(suite_name='pure_consumer') + self.assertEqual( + _suites_with_register_consts([sr]), + [], + ) + + +class TestModuleSkippedWhenNoConstituents(unittest.TestCase): + """``_generate_host_constituents`` returns ``None`` when nothing touches + constituent state — the module is not emitted at all.""" + + def test_returns_none(self): + suite_resolution, _hd = _resolve_simple() + self.assertIsNone(_generate_host_constituents([suite_resolution])) + + +class TestModuleHeaderAndUses(unittest.TestCase): + """Module declaration and external USEs are correct.""" + + def setUp(self): + self.text = _render_consumer() + + def test_module_name(self): + self.assertIn('module ccpp_host_constituents', self.text) + self.assertIn('end module ccpp_host_constituents', self.text) + + def test_use_kind(self): + self.assertIn('use ccpp_kinds, only: kind_phys', self.text) + + def test_use_constituent_prop_mod(self): + self.assertIn('use ccpp_constituent_prop_mod', self.text) + self.assertIn('ccpp_model_constituents_t', self.text) + self.assertIn('ccpp_constituent_properties_t', self.text) + self.assertIn('ccpp_constituent_prop_ptr_t', self.text) + + +class TestStateDeclarations(unittest.TestCase): + """Module-level state declarations: obj, pointers, integers.""" + + def setUp(self): + self.consumer_text = '\n'.join( + _render_consumer().splitlines() + ) + self.register_text = '\n'.join( + _render_register().splitlines() + ) + + def test_constituent_obj_declared(self): + # Per-instance allocatable array. + self.assertIn( + 'type(ccpp_model_constituents_t), target, allocatable :: ' + 'ccpp_model_constituents_obj(:)', + self.consumer_text, + ) + + def test_obj_is_public(self): + self.assertIn('public :: ccpp_model_constituents_obj', self.consumer_text) + + def test_no_module_level_pointers(self): + # Under per-instance design, ccpp_constituents / ..._tendencies / + # ..._properties / number_of_ccpp_constituents are NOT module-level + # variables — they're accessed as members of the obj(inst) array. + self.assertNotIn('pointer :: ccpp_constituents', self.consumer_text) + self.assertNotIn( + 'pointer :: ccpp_constituent_tendencies', self.consumer_text, + ) + self.assertNotIn( + 'pointer :: ccpp_constituent_properties', self.consumer_text, + ) + self.assertNotIn( + 'integer :: number_of_ccpp_constituents', self.consumer_text, + ) + + def test_index_of_X_declared(self): + self.assertIn( + 'integer :: index_of_cloud_liquid_water_mixing_ratio = int_unassigned', + self.consumer_text, + ) + self.assertIn( + 'public :: index_of_cloud_liquid_water_mixing_ratio', + self.consumer_text, + ) + + def test_per_suite_buffer_declared_for_producer(self): + # The per-suite buffer is a per-instance wrapper-DDT array so each + # instance owns its own scheme-registered constituent property + # objects (the wrapper type is emitted once at module scope). + self.assertIn('type :: ccpp_dyn_const_buffer_t', self.register_text) + self.assertIn( + 'type(ccpp_constituent_properties_t), allocatable :: items(:)', + self.register_text, + ) + self.assertIn( + 'type(ccpp_dyn_const_buffer_t), allocatable, target :: ' + 'reg_consts_dynamic_constituents(:)', + self.register_text, + ) + self.assertIn( + 'public :: reg_consts_dynamic_constituents', + self.register_text, + ) + + def test_no_per_suite_buffer_when_no_producer(self): + # consumer fixture has no register-phase producer schemes — no + # ``_dynamic_constituents`` array declaration, no public + # of any such buffer. (The ccpp_deallocate_dynamic_constituents + # subroutine name is unrelated and is expected to appear.) + self.assertNotIn('allocatable, target :: ', self.consumer_text) + self.assertNotIn('_dynamic_constituents(:)', self.consumer_text) + + +class TestRegisterConstituentsRoutine(unittest.TestCase): + """``ccpp_register_constituents`` merges host + per-suite buffers.""" + + def setUp(self): + # Use a SuiteResolution list containing BOTH consumer and register + # fixtures so the routine iterates a real per-suite buffer. + self.text = _render_register() + + def test_takes_host_constituents_and_instance(self): + # instance_number AND number_of_instances are both in the signature + # when the host declares the multi-instance pair. + self.assertIn( + 'subroutine ccpp_register_constituents(host_constituents, ' + 'inst_num, ninstances, errcode, errmsg)', + self.text, + ) + self.assertIn( + 'type(ccpp_constituent_properties_t), target, intent(in) :: ' + 'host_constituents(:)', + self.text, + ) + self.assertIn('integer, intent(in) :: inst_num', self.text) + + def test_allocates_obj_array_on_first_call(self): + # Idempotent allocation: only the first instance to call sees + # an unallocated array; subsequent instances skip. + self.assertIn( + 'if (.not. allocated(ccpp_model_constituents_obj)) then', + self.text, + ) + self.assertIn( + 'allocate(ccpp_model_constituents_obj(ninstances))', self.text, + ) + + def test_initializes_table_per_instance(self): + self.assertIn( + 'call ccpp_model_constituents_obj(inst_num)%initialize_table(num_consts)', + self.text, + ) + # Count of scheme-registered constituents comes from THIS instance's + # slot in the wrapper-DDT array, not the (no-longer-shared) buffer. + self.assertIn( + 'num_consts = num_consts + size(' + 'reg_consts_dynamic_constituents(inst_num)%items, 1)', + self.text, + ) + + def test_iterates_host_then_suite_per_instance(self): + body = self.text.split('subroutine ccpp_register_constituents')[1].split( + 'end subroutine ccpp_register_constituents' + )[0] + host_pos = body.find('host_constituents(index)') + suite_pos = body.find( + 'reg_consts_dynamic_constituents(inst_num)%items(index)' + ) + self.assertGreater(host_pos, 0) + self.assertGreater(suite_pos, host_pos) + # All %new_field calls go through obj(inst_num). + self.assertIn( + 'call ccpp_model_constituents_obj(inst_num)%new_field(const_prop', + body, + ) + + def test_lock_table_called_per_instance(self): + self.assertIn( + 'call ccpp_model_constituents_obj(inst_num)%lock_table(' + 'errcode=errcode, errmsg=errmsg)', + self.text, + ) + + +class TestInitializeConstituentsRoutine(unittest.TestCase): + """``ccpp_initialize_constituents`` locks data + binds pointers + populates indices.""" + + def setUp(self): + self.text = _render_consumer() + + def test_takes_dimensions_and_instance(self): + self.assertIn( + 'subroutine ccpp_initialize_constituents(ncols, num_layers, ' + 'inst_num, errcode, errmsg)', + self.text, + ) + + def test_calls_lock_data_per_instance(self): + self.assertIn( + 'call ccpp_model_constituents_obj(inst_num)%lock_data(' + 'ncols, num_layers, errcode=errcode, errmsg=errmsg)', + self.text, + ) + + def test_uses_scheme_utils(self): + body = self.text.split('subroutine ccpp_initialize_constituents')[1].split( + 'end subroutine ccpp_initialize_constituents' + )[0] + self.assertIn( + 'use ccpp_scheme_utils, only: ccpp_initialize_constituent_ptr', + body, + ) + + def test_initialises_scheme_utils_pointer_per_instance(self): + body = self.text.split('subroutine ccpp_initialize_constituents')[1].split( + 'end subroutine ccpp_initialize_constituents' + )[0] + # Need a local pointer variable to pass a non-pointer target as a + # pointer dummy. + self.assertIn( + 'type(ccpp_model_constituents_t), pointer :: const_obj_ptr', + body, + ) + self.assertIn( + 'const_obj_ptr => ccpp_model_constituents_obj(inst_num)', body, + ) + self.assertIn( + 'call ccpp_initialize_constituent_ptr(const_obj_ptr)', body, + ) + + def test_no_module_pointer_binding(self): + # The old module-level ccpp_constituents pointers don't exist; + # this routine must NOT try to bind them. + body = self.text.split('subroutine ccpp_initialize_constituents')[1].split( + 'end subroutine ccpp_initialize_constituents' + )[0] + self.assertNotIn('ccpp_constituents =>', body) + self.assertNotIn('ccpp_constituent_tendencies =>', body) + self.assertNotIn('ccpp_constituent_properties =>', body) + self.assertNotIn( + 'number_of_ccpp_constituents =', body, + ) + + def test_queries_index_of_X_per_instance(self): + self.assertIn( + "call ccpp_model_constituents_obj(inst_num)%const_index(" + "index_of_cloud_liquid_water_mixing_ratio, " + "'cloud_liquid_water_mixing_ratio', errcode=errcode, errmsg=errmsg)", + self.text, + ) + + def test_int_unassigned_imported(self): + # Used by the post-const_index validation block; must be in the + # module-level USE so the contained subroutine can reference it. + self.assertIn( + 'use ccpp_constituent_prop_mod, only:', self.text, + ) + self.assertIn('int_unassigned', self.text) + + def test_post_const_index_validation_emitted(self): + # %const_index doesn't error on a miss — it returns int_unassigned + # and errcode=0. The generator must check the integer afterward + # and fail with a descriptive message so the host sees the bad + # registration at init time instead of crashing on a -huge(1) + # subscript later in run-phase scheme calls. + body = self.text.split('subroutine ccpp_initialize_constituents')[1].split( + 'end subroutine ccpp_initialize_constituents' + )[0] + self.assertIn( + 'if (index_of_cloud_liquid_water_mixing_ratio == int_unassigned) then', + body, + ) + self.assertIn('errcode = 1', body) + self.assertIn( + "errmsg = 'ccpp_initialize_constituents: constituent " + "''cloud_liquid_water_mixing_ratio'' is referenced by a " + "scheme but is not in the registered constituent table", + body, + ) + + +def _render_long_name_constituent(): + """Resolve a single-suite scheme that consumes ONE advected base + constituent whose standard name is long enough to force + ``_index_symbol_name`` to mangle the Fortran symbol, then render the + host-constituents module. Returns ``(text, long_std_name, symbol)``. + + Regression guard for the lossy round-trip bug: the mangled + ``index_of_`` Fortran symbol must NOT leak into the ``%const_index`` + lookup string (the framework keys on the real standard name) nor into + ``ccpp_model_const_stdnames``. + """ + import tempfile + import logging + from metadata.metadata_table import parse_metadata_file + from metadata.variable_resolver import build_flat_host_dict, SchemeStore + from generator.suite_xml import parse_suite_xml + + # 57 chars -> index_of_ is 66 > 63 -> mangled (same family as the + # cam-sima kessler ``*_wrt_moist_air_and_condensed_water`` constituents). + long_std = 'water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water' + from generator.suite_resolver import _index_symbol_name + symbol = _index_symbol_name(long_std) + assert symbol != 'index_of_' + long_std, 'fixture name must mangle' + + scheme_meta = ''' +[ccpp-table-properties] + name = long_const_scheme + type = scheme +[ccpp-arg-table] + name = long_const_scheme_run + type = scheme +[ ncol ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + intent = in +[ nz ] + standard_name = vertical_layer_dimension + units = count + dimensions = () + type = integer + intent = in +[ qv ] + standard_name = %s + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = in + advected = .true. +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' % long_std + suite_xml = ( + '\n' + '\n' + ' \n' + ' long_const_scheme\n' + ' \n' + '\n' + ) + + here = os.path.dirname(os.path.abspath(__file__)) + samples = os.path.join(here, 'sample_files') + host_tbls = parse_metadata_file(os.path.join(samples, 'host_with_constituents.meta')) + ctrl_tbls = parse_metadata_file(os.path.join(samples, 'control_full.meta')) + fw_meta = os.path.join( + os.path.dirname(here), 'capgen', 'src', 'ccpp_constituent_prop_mod.meta', + ) + ddt_tbls = parse_metadata_file(fw_meta) if os.path.isfile(fw_meta) else [] + hd = build_flat_host_dict(host_tbls, ctrl_tbls, ddt_tbls) + + with tempfile.TemporaryDirectory() as tmp: + sm = os.path.join(tmp, 'long_const_scheme.meta') + with open(sm, 'w') as fh: + fh.write(scheme_meta) + sx = os.path.join(tmp, 'suite_longc.xml') + with open(sx, 'w') as fh: + fh.write(suite_xml) + store = SchemeStore.build_from(parse_metadata_file(sm)) + suite = parse_suite_xml(sx, tmp, logging.getLogger('test'), + skip_validation=True) + sr = resolve_suite(suite, store, hd) + text = '\n'.join(_generate_host_constituents([sr], host_dict=hd)) + return text, long_std, symbol + + +class TestLongConstituentNameRoundTrip(unittest.TestCase): + """Regression: a constituent std name long enough to mangle the Fortran + ``index_of_`` symbol must still key ``%const_index`` (and the + ``ccpp_model_const_stdnames`` array) on the REAL standard name. The old + code recovered the name from the mangled symbol suffix, so the lookup + string was corrupted, ``const_index`` never matched, and the index + integer stayed at its ``0`` default -> out-of-bounds constituent + subscript at run time (cam-sima kessler segfault).""" + + @classmethod + def setUpClass(cls): + cls.text, cls.long_std, cls.symbol = _render_long_name_constituent() + + def test_const_index_keys_on_real_std_name(self): + # Mangled symbol as the integer; REAL std name as the lookup string. + self.assertIn( + "%const_index({}, '{}', errcode=errcode, errmsg=errmsg)".format( + self.symbol, self.long_std, + ), + self.text, + ) + + def test_mangled_suffix_not_used_as_lookup_string(self): + # The mangled suffix must never appear as a quoted lookup key. + mangled_suffix = self.symbol[len('index_of_'):] + self.assertNotIn("'{}'".format(mangled_suffix), self.text) + + def test_stdnames_array_lists_real_name(self): + self.assertIn("'{}'".format(self.long_std), self.text) + + def test_subscript_symbol_declared_and_public(self): + # The (mangled) symbol must still be declared, public, and reset. + self.assertIn( + 'integer :: {} = int_unassigned'.format(self.symbol), self.text) + self.assertIn('public :: {}'.format(self.symbol), self.text) + + def test_index_defaults_and_resets_to_unassigned(self): + # Declaration default is the unbound sentinel (so a never-bound index + # trips the init guard instead of becoming a 0 subscript)... + self.assertIn( + 'integer :: {} = int_unassigned'.format(self.symbol), self.text) + # ...and the deallocate routine resets it to the same sentinel. + dealloc = self.text.split( + 'subroutine ccpp_deallocate_dynamic_constituents' + )[1].split('end subroutine ccpp_deallocate_dynamic_constituents')[0] + self.assertIn('{} = int_unassigned'.format(self.symbol), dealloc) + self.assertNotIn('{} = 0'.format(self.symbol), dealloc) + + +class TestIsSchemeConstituent(unittest.TestCase): + """``ccpp_is_scheme_constituent`` + the module-scope std-name parameter array.""" + + def setUp(self): + self.text = _render_consumer() + + def test_subroutine_signature(self): + self.assertIn( + 'subroutine ccpp_is_scheme_constituent(var_name, ' + 'constituent_exists, errcode, errmsg)', + self.text, + ) + + def test_uses_known_array(self): + self.assertIn( + 'constituent_exists = any(ccpp_model_const_stdnames == var_name)', + self.text, + ) + + def test_param_array_declared(self): + self.assertIn( + "character(len=31), parameter :: ccpp_model_const_stdnames(1) = (/ &", + self.text, + ) + self.assertIn("'cloud_liquid_water_mixing_ratio'", self.text) + + def test_param_array_public(self): + self.assertIn('public :: ccpp_model_const_stdnames', self.text) + + +class TestIsSchemeConstituentNoIndices(unittest.TestCase): + """When the suite registers constituents but none are referenced via + ``index_of_``, ``ccpp_is_scheme_constituent`` returns ``.false.`` + unconditionally — no parameter array is emitted.""" + + def setUp(self): + self.text = _render_register() + + def test_falls_back_to_constant_false(self): + self.assertIn('constituent_exists = .false.', self.text) + + def test_no_param_array(self): + self.assertNotIn('ccpp_model_const_stdnames', self.text) + + +class TestWrapperSubroutines(unittest.TestCase): + """Thin wrappers for number / gather / update / const_index.""" + + def setUp(self): + self.text = _render_consumer() + + def test_number_constituents(self): + self.assertIn( + 'subroutine ccpp_number_constituents(num_flds, advected, ' + 'inst_num, errcode, errmsg)', + self.text, + ) + self.assertIn( + 'call ccpp_model_constituents_obj(inst_num)%num_constituents(' + 'num_flds, advected=advected, errcode=errcode, errmsg=errmsg)', + self.text, + ) + + def test_gather_constituents(self): + self.assertIn( + 'call ccpp_model_constituents_obj(inst_num)%copy_in(' + 'const_array, errcode=errcode, errmsg=errmsg)', + self.text, + ) + + def test_update_constituents(self): + self.assertIn( + 'call ccpp_model_constituents_obj(inst_num)%copy_out(' + 'const_array, errcode=errcode, errmsg=errmsg)', + self.text, + ) + + def test_const_get_index(self): + # Keyword args ensure unambiguous mapping to the DDT's signature + # (index, standard_name, errcode, errmsg). + self.assertIn( + 'call ccpp_model_constituents_obj(inst_num)%const_index(' + 'standard_name=stdname, index=const_index, ' + 'errcode=errcode, errmsg=errmsg)', + self.text, + ) + + +class TestAccessorFunctions(unittest.TestCase): + """Pointer-returning accessor functions.""" + + def setUp(self): + self.text = _render_consumer() + + def test_constituents_array(self): + self.assertIn( + 'function ccpp_constituents_array(inst_num) result(const_ptr)', + self.text, + ) + self.assertIn( + 'const_ptr => ccpp_model_constituents_obj(inst_num)%field_data_ptr()', + self.text, + ) + + def test_advected_constituents_array(self): + self.assertIn( + 'function ccpp_advected_constituents_array(inst_num) result(const_ptr)', + self.text, + ) + self.assertIn( + 'const_ptr => ccpp_model_constituents_obj(inst_num)' + '%advected_constituents_ptr()', + self.text, + ) + + def test_model_const_properties(self): + self.assertIn( + 'function ccpp_model_const_properties(inst_num) result(const_ptr)', + self.text, + ) + self.assertIn( + 'const_ptr => ccpp_model_constituents_obj(inst_num)' + '%constituent_props_ptr()', + self.text, + ) + + +class TestDeallocateRoutine(unittest.TestCase): + """``ccpp_deallocate_dynamic_constituents`` is per-instance with + last-to-leave teardown of shared buffers + the obj array.""" + + def setUp(self): + self.text = _render_register() + self.body = self.text.split( + 'subroutine ccpp_deallocate_dynamic_constituents' + )[1].split('end subroutine ccpp_deallocate_dynamic_constituents')[0] + + def test_takes_instance_number(self): + self.assertIn( + 'subroutine ccpp_deallocate_dynamic_constituents(inst_num)', + self.text, + ) + self.assertIn('integer, intent(in) :: inst_num', self.body) + + def test_per_instance_reset(self): + self.assertIn( + 'call ccpp_model_constituents_obj(inst_num)%reset()', self.body, + ) + + def test_short_circuit_when_unallocated(self): + # If no instance has registered yet, the call is a no-op. + self.assertIn( + 'if (.not. allocated(ccpp_model_constituents_obj)) return', + self.body, + ) + + def test_last_to_leave_check(self): + self.assertIn('all_done = .true.', self.body) + self.assertIn('do i = 1, size(ccpp_model_constituents_obj, 1)', self.body) + self.assertIn( + 'if (ccpp_model_constituents_obj(i)%const_props_locked()) then', + self.body, + ) + self.assertIn('all_done = .false.', self.body) + + def test_last_to_leave_teardown(self): + self.assertIn('if (all_done) then', self.body) + self.assertIn('deallocate(ccpp_model_constituents_obj)', self.body) + # The per-suite buffer is NOT torn down here — that's owned by + # the suite-cap lifecycle (deallocated in _final's + # last-to-leave block). Tearing it down here would break the + # next ccpp_register call (suite_state guard skips re-fill). + self.assertNotIn( + 'deallocate(reg_consts_dynamic_constituents)', self.body, + ) + + def test_no_module_pointer_nullify(self): + # The module-level pointer set was removed; deallocate routine + # must not try to nullify them. + self.assertNotIn('nullify(ccpp_constituents)', self.body) + self.assertNotIn('nullify(ccpp_constituent_tendencies)', self.body) + self.assertNotIn('nullify(ccpp_constituent_properties)', self.body) + self.assertNotIn('number_of_ccpp_constituents = 0', self.body) + + +def load_tests(loader, tests, ignore): + import generator.host_constituents as hc + tests.addTests(doctest.DocTestSuite(hc)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_integration.py b/unit-tests/test_integration.py new file mode 100644 index 00000000..eecd7463 --- /dev/null +++ b/unit-tests/test_integration.py @@ -0,0 +1,2325 @@ +"""End-to-end integration tests for capgen.capgen(). + +These tests invoke the full pipeline — metadata loading, variable resolution, +and file generation — and verify that all expected output files are produced +with correct content. No Fortran compiler is involved; correctness is checked +at the text/XML level. +""" + +import os +import tempfile +import time +import unittest +import xml.etree.ElementTree as ET + +from ccpp_capgen import capgen + +_TESTS_DIR = os.path.dirname(__file__) +_SAMPLES_DIR = os.path.join(_TESTS_DIR, 'sample_files') +_SUITE_DIR = os.path.join(_TESTS_DIR, 'sample_suite_files') + + +def _sf(name): + return os.path.join(_SAMPLES_DIR, name) + + +def _suite_file(name): + return os.path.join(_SUITE_DIR, name) + + +# --------------------------------------------------------------------------- +# Helper: run capgen and return output directory + file map +# --------------------------------------------------------------------------- + +def _run_simple(tmpdir, suite_xml='suite_test_simple.xml', kind_types=None): + """Run capgen with the simple test suite and return the output dir.""" + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[_sf('scheme_multipart.meta')], + suite_files=[_suite_file(suite_xml)], + output_root=tmpdir, + kind_types=kind_types or {}, + ) + return tmpdir + + +def _run_subcycle(tmpdir): + return _run_simple(tmpdir, suite_xml='suite_test_subcycle.xml') + + +# --------------------------------------------------------------------------- +# Test: output files exist +# --------------------------------------------------------------------------- + +class TestOutputFilesExist(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def _path(self, name): + return os.path.join(self._tmpdir, name) + + def test_host_cap_exists(self): + self.assertTrue(os.path.isfile(self._path('test_host_ccpp_cap.F90'))) + + def test_suite_cap_exists(self): + self.assertTrue(os.path.isfile(self._path('ccpp_test_simple_cap.F90'))) + + def test_group_cap_exists(self): + self.assertTrue(os.path.isfile(self._path('ccpp_test_simple_physics_cap.F90'))) + + def test_suite_data_exists(self): + self.assertTrue(os.path.isfile(self._path('ccpp_test_simple_data.F90'))) + + def test_datatable_exists(self): + self.assertTrue(os.path.isfile(self._path('datatable.xml'))) + + def test_ccpp_kinds_always_generated(self): + # ccpp_kinds.F90 is always written, even when no --kind-type is given. + self.assertTrue(os.path.isfile(self._path('ccpp_kinds.F90'))) + + def test_ccpp_kinds_default_kind_phys(self): + """With no --kind-type, kind_phys defaults to REAL64 from iso_fortran_env.""" + with open(self._path('ccpp_kinds.F90')) as fh: + text = fh.read() + self.assertIn('use iso_fortran_env, only: REAL64', text) + self.assertIn('kind_phys = REAL64', text) + + def test_ccpp_kinds_with_explicit_kind_types(self): + with tempfile.TemporaryDirectory() as d: + _run_simple(d, kind_types={ + 'kind_phys': ('iso_fortran_env', 'REAL64'), + }) + self.assertTrue(os.path.isfile(os.path.join(d, 'ccpp_kinds.F90'))) + + def test_ccpp_kinds_default_injected_when_other_kinds_given(self): + """kind_phys default is injected even when other --kind-type args are present.""" + with tempfile.TemporaryDirectory() as d: + _run_simple(d, kind_types={ + 'kind_dyn': ('iso_fortran_env', 'REAL32'), + }) + with open(os.path.join(d, 'ccpp_kinds.F90')) as fh: + text = fh.read() + self.assertIn('kind_dyn', text) + self.assertIn('kind_phys', text) + + +# --------------------------------------------------------------------------- +# Test: kind_spec declared in metadata is folded into ccpp_kinds.F90 +# --------------------------------------------------------------------------- + +def _inject_kind_spec_lines(src_meta, dest_meta, lines): + """Copy *src_meta* to *dest_meta*, inserting *lines* into the first + ``[ccpp-table-properties]`` block immediately after its header.""" + with open(src_meta) as fh: + text = fh.read() + marker = '[ccpp-table-properties]' + idx = text.find(marker) + if idx < 0: + raise RuntimeError("no ccpp-table-properties marker in " + src_meta) + nl = text.find('\n', idx) + insertion = ''.join(' ' + line + '\n' for line in lines) + new_text = text[:nl + 1] + insertion + text[nl + 1:] + with open(dest_meta, 'w') as fh: + fh.write(new_text) + + +class TestMetadataKindSpec(unittest.TestCase): + """Integration tests for kind_spec declared in [ccpp-table-properties].""" + + def _scheme_with_kind_spec(self, dest_dir, lines): + """Write a copy of scheme_multipart.meta with extra ``kind_spec`` lines.""" + dest = os.path.join(dest_dir, 'scheme_multipart.meta') + _inject_kind_spec_lines(_sf('scheme_multipart.meta'), dest, lines) + return dest + + def test_metadata_kind_spec_added_to_ccpp_kinds(self): + """A ``kind_spec`` declared in scheme metadata appears in ccpp_kinds.F90.""" + with tempfile.TemporaryDirectory() as d: + scheme_meta = self._scheme_with_kind_spec( + d, ['kind_spec = temp_kinds:kind_temp=>temp_r8'], + ) + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[scheme_meta], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=d, + kind_types={}, + ) + with open(os.path.join(d, 'ccpp_kinds.F90')) as fh: + text = fh.read() + # Use lines may be column-aligned when multiple modules are + # present; match each token rather than the full line. + self.assertRegex(text, r'use\s+temp_kinds\b[^\n]*only:\s*temp_r8') + self.assertRegex(text, r'kind_temp\s*=\s*temp_r8') + # kind_phys default still injected. + self.assertIn('kind_phys', text) + self.assertIn('use iso_fortran_env', text) + + def test_metadata_kind_spec_shorthand_added_to_ccpp_kinds(self): + """Shorthand ``module:spec`` republishes spec under its own name.""" + with tempfile.TemporaryDirectory() as d: + scheme_meta = self._scheme_with_kind_spec( + d, ['kind_spec = host_kinds:kind_r4'], + ) + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[scheme_meta], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=d, + kind_types={}, + ) + with open(os.path.join(d, 'ccpp_kinds.F90')) as fh: + text = fh.read() + self.assertRegex(text, r'use\s+host_kinds\b[^\n]*only:\s*kind_r4') + self.assertRegex(text, r'kind_r4\s*=\s*kind_r4') + + def test_cli_and_metadata_identical_kind_spec_no_conflict(self): + """CLI --kind-type and metadata kind_spec for the same kind are accepted when identical.""" + with tempfile.TemporaryDirectory() as d: + scheme_meta = self._scheme_with_kind_spec( + d, ['kind_spec = temp_kinds:kind_temp=>temp_r8'], + ) + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[scheme_meta], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=d, + kind_types={'kind_temp': ('temp_kinds', 'temp_r8')}, + ) + with open(os.path.join(d, 'ccpp_kinds.F90')) as fh: + text = fh.read() + # The kind appears exactly once in ccpp_kinds.F90. + self.assertEqual(text.count('kind_temp ='), 1) + + def test_cli_and_metadata_conflicting_kind_spec_raises(self): + """CLI and metadata declaring the same kind with different specs is a hard error.""" + with tempfile.TemporaryDirectory() as d: + scheme_meta = self._scheme_with_kind_spec( + d, ['kind_spec = temp_kinds:kind_temp=>temp_r8'], + ) + from metadata.parse_tools import CCPPError + with self.assertRaises(CCPPError) as cm: + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[scheme_meta], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=d, + kind_types={'kind_temp': ('other_kinds', 'r8')}, + ) + self.assertIn('kind_temp', str(cm.exception)) + + +# --------------------------------------------------------------------------- +# Test: host_constituents module + framework file inclusion +# --------------------------------------------------------------------------- + +def _run_with_constituents(tmpdir, suite_xml='suite_consume_constituent.xml'): + """Run capgen with a constituent-using fixture and return tmpdir.""" + capgen( + host_name='test_host', + host_files=[_sf('host_with_constituents.meta'), + _sf('control_full.meta')], + scheme_files=[_sf('scheme_consume_constituent.meta')], + suite_files=[_suite_file(suite_xml)], + output_root=tmpdir, + kind_types={}, + ) + return tmpdir + + +class TestHostConstituentsEmittedEndToEnd(unittest.TestCase): + """Full pipeline emits ccpp_host_constituents.F90 and lists every + framework F90 dependency in datatable.xml's .""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_with_constituents(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_host_constituents_file_exists(self): + self.assertTrue(os.path.isfile( + os.path.join(self._tmpdir, 'ccpp_host_constituents.F90') + )) + + def _utility_paths(self): + tree = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')) + return [e.text for e in tree.getroot().findall( + './capgen_files/utilities/file' + )] + + def test_datatable_lists_host_constituents(self): + utils = self._utility_paths() + names = [os.path.basename(p) for p in utils] + self.assertIn('ccpp_host_constituents.F90', names) + + def test_datatable_lists_framework_constituent_module(self): + utils = self._utility_paths() + names = [os.path.basename(p) for p in utils] + self.assertIn('ccpp_constituent_prop_mod.F90', names) + + def test_datatable_lists_framework_dependencies(self): + utils = self._utility_paths() + names = [os.path.basename(p) for p in utils] + # Transitive deps of ccpp_constituent_prop_mod. + self.assertIn('ccpp_hashable.F90', names) + self.assertIn('ccpp_hash_table.F90', names) + # Used by cam-sima schemes (ccpp_constituent_index). + self.assertIn('ccpp_scheme_utils.F90', names) + + def test_framework_paths_absolute_and_existing(self): + utils = self._utility_paths() + framework_names = { + 'ccpp_constituent_prop_mod.F90', + 'ccpp_hashable.F90', + 'ccpp_hash_table.F90', + 'ccpp_scheme_utils.F90', + } + for path in utils: + if os.path.basename(path) in framework_names: + self.assertTrue(os.path.isabs(path), + 'framework path not absolute: ' + path) + self.assertTrue(os.path.isfile(path), + 'framework file missing: ' + path) + + def test_framework_paths_resolve_under_capgen_src(self): + """Every framework F90 listed must resolve under capgen/src/. + Capgen ships self-contained — no parent-dir fallback. If any + framework file lands outside capgen/src/ this test fails so + downstream consumers (vendoring just capgen/) don't silently + miss a required dependency.""" + from ccpp_capgen import _FRAMEWORK_SRC_DIR + framework_names = { + 'ccpp_constituent_prop_mod.F90', + 'ccpp_hashable.F90', + 'ccpp_hash_table.F90', + 'ccpp_scheme_utils.F90', + } + utils = self._utility_paths() + canonical = os.path.abspath(_FRAMEWORK_SRC_DIR) + for path in utils: + if os.path.basename(path) in framework_names: + self.assertEqual( + os.path.abspath(os.path.dirname(path)), canonical, + 'framework F90 outside capgen/src/: ' + path, + ) + + +class TestResolveFrameworkF90FilesMissingRaises(unittest.TestCase): + """``_resolve_framework_f90_files`` raises CCPPError listing the + missing file(s) when a required framework F90 is not present under + capgen/src/. Catches deployment errors immediately instead of + leaving the host build to fail with an opaque "Cannot open module + file" message at compile time.""" + + def test_missing_file_raises_with_actionable_message(self): + import ccpp_capgen + from metadata.parse_tools import CCPPError + # Append a never-vendored sentinel to the framework-F90 list, + # then restore on teardown via a try/finally so we don't leak + # state into other tests. + original = list(ccpp_capgen._FRAMEWORK_F90_FILES) + ccpp_capgen._FRAMEWORK_F90_FILES.append('definitely_missing.F90') + try: + with self.assertRaises(CCPPError) as cm: + ccpp_capgen._resolve_framework_f90_files() + finally: + ccpp_capgen._FRAMEWORK_F90_FILES[:] = original + msg = str(cm.exception) + # Names the missing file, the search dir, and what to do. + self.assertIn('definitely_missing.F90', msg) + self.assertIn(ccpp_capgen._FRAMEWORK_SRC_DIR, msg) + self.assertIn('Vendor', msg) + + +class TestNoHostConstituentsWhenAbsent(unittest.TestCase): + """The host-constituents module is NOT emitted (and the framework F90 + files are NOT listed) when no suite touches constituent state.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_host_constituents_file_absent(self): + self.assertFalse(os.path.isfile( + os.path.join(self._tmpdir, 'ccpp_host_constituents.F90') + )) + + def test_no_framework_dependencies_in_utilities(self): + tree = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')) + utils = [e.text for e in tree.getroot().findall( + './capgen_files/utilities/file' + )] + names = [os.path.basename(p) for p in utils] + # Only ccpp_kinds.F90 — no constituent framework files. + self.assertEqual(names, ['ccpp_kinds.F90']) + + +# --------------------------------------------------------------------------- +# Test: static API content +# --------------------------------------------------------------------------- + +class TestHostCapContent(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + with open(os.path.join(self._tmpdir, 'test_host_ccpp_cap.F90')) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_module_declaration(self): + self.assertIn('module test_host_ccpp_cap', self.text) + + def test_ccpp_register_always_present(self): + # ccpp_register is mandatory in the new design and always emitted, + # even when no scheme has a register phase (state transition only). + self.assertIn('subroutine ccpp_register', self.text) + + def test_ccpp_init_present(self): + self.assertIn('subroutine ccpp_init', self.text) + + def test_ccpp_final_present(self): + self.assertIn('subroutine ccpp_final', self.text) + + def test_ccpp_physics_run_present(self): + self.assertIn('subroutine ccpp_physics_run', self.text) + + def test_dispatches_to_test_simple(self): + self.assertIn("case('test_simple')", self.text) + + def test_uses_suite_cap_module(self): + self.assertIn('use ccpp_test_simple_cap', self.text) + + +# --------------------------------------------------------------------------- +# Test: suite cap content +# --------------------------------------------------------------------------- + +class TestSuiteCapContent(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_module_declaration(self): + self.assertIn('module ccpp_test_simple_cap', self.text) + + def test_register_subroutine_always_present(self): + # _register is mandatory in the new design — emitted even when + # no scheme has a register phase (state-transition skeleton only). + self.assertIn('subroutine test_simple_register', self.text) + + def test_init_subroutine(self): + self.assertIn('subroutine test_simple_init', self.text) + + def test_final_subroutine(self): + self.assertIn('subroutine test_simple_final', self.text) + + def test_physics_run_subroutine(self): + self.assertIn('subroutine test_simple_physics_run', self.text) + + def test_dispatches_to_group(self): + self.assertIn("case('physics')", self.text) + + def test_state_alloc_called(self): + self.assertIn('state_alloc', self.text) + + def test_state_dealloc_called(self): + self.assertIn('state_dealloc', self.text) + + +# --------------------------------------------------------------------------- +# Test: group cap content +# --------------------------------------------------------------------------- + +class TestGroupCapContent(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_module_declaration(self): + self.assertIn('module ccpp_test_simple_physics_cap', self.text) + + def test_state_machine_params(self): + self.assertIn('CCPP_GROUP_UNINITIALIZED', self.text) + self.assertIn('CCPP_GROUP_INITIALIZED', self.text) + + def test_scheme_call_present(self): + self.assertIn('call temp_calc_adjust_run', self.text) + + def test_init_guard(self): + self.assertIn('CCPP_GROUP_INITIALIZED', self.text) + self.assertIn('ccpp_group_state', self.text) + + def test_state_alloc_subroutine(self): + self.assertIn('subroutine physics_state_alloc', self.text) + + def test_state_dealloc_subroutine(self): + self.assertIn('subroutine physics_state_dealloc', self.text) + + def test_ends_with_newline(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: + raw = fh.read() + self.assertTrue(raw.endswith('\n')) + + +# --------------------------------------------------------------------------- +# Test: datatable.xml content +# --------------------------------------------------------------------------- + +class TestDatatableContent(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + self._root = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_root_version(self): + self.assertEqual(self._root.get('version'), '1.0') + + def test_suite_file_in_capgen_files(self): + suite_files = self._root.find('capgen_files').find('suite_files') + names = [os.path.basename(f.text) for f in suite_files.findall('file')] + self.assertIn('ccpp_test_simple_cap.F90', names) + self.assertIn('ccpp_test_simple_physics_cap.F90', names) + + def test_host_cap_in_host_files(self): + host_files = self._root.find('capgen_files').find('host_files') + self.assertIsNotNone(host_files) + names = [os.path.basename(f.text) for f in host_files.findall('file')] + self.assertIn('test_host_ccpp_cap.F90', names) + + def test_ccpp_kinds_in_utilities(self): + # ccpp_kinds.F90 is always generated and must be discoverable by + # CMake via (matches the original ccpp_capgen behavior). + utils = self._root.find('capgen_files').find('utilities') + names = [os.path.basename(f.text) for f in utils.findall('file')] + self.assertIn('ccpp_kinds.F90', names) + + def test_ccpp_kinds_utilities_path_is_absolute_and_exists(self): + """The path stored in must be absolute and resolve to a real file.""" + utils = self._root.find('capgen_files').find('utilities') + kinds_paths = [ + f.text for f in utils.findall('file') + if os.path.basename(f.text) == 'ccpp_kinds.F90' + ] + self.assertEqual(len(kinds_paths), 1) + self.assertTrue(os.path.isabs(kinds_paths[0])) + self.assertTrue(os.path.isfile(kinds_paths[0])) + + def test_suite_meta_in_inspection_files(self): + inspection = self._root.find('inspection_files') + self.assertIsNotNone(inspection) + meta_files = inspection.find('suite_meta_files') + self.assertIsNotNone(meta_files) + names = [os.path.basename(f.text) for f in meta_files.findall('file')] + self.assertIn('ccpp_test_simple_data.meta', names) + + def test_suite_meta_not_in_capgen_files(self): + capgen_files = self._root.find('capgen_files') + self.assertIsNone(capgen_files.find('suite_meta_files')) + + def test_expanded_sdf_in_inspection_files(self): + inspection = self._root.find('inspection_files') + self.assertIsNotNone(inspection) + exp_files = inspection.find('expanded_sdf_files') + self.assertIsNotNone(exp_files) + names = [os.path.basename(f.text) for f in exp_files.findall('file')] + self.assertIn('ccpp_test_simple_expanded.xml', names) + # The path stored must resolve to a real file on disk. + for f in exp_files.findall('file'): + self.assertTrue(os.path.isabs(f.text)) + self.assertTrue(os.path.isfile(f.text)) + + def test_scheme_in_schemes(self): + names = [s.get('name') for s in self._root.find('schemes').findall('scheme')] + self.assertIn('temp_calc_adjust', names) + + def test_suite_in_api(self): + suites = self._root.find('api').find('suites') + names = [s.get('name') for s in suites.findall('suite')] + self.assertIn('test_simple', names) + + def test_group_in_api_suite(self): + suites = self._root.find('api').find('suites') + suite = next(s for s in suites.findall('suite') if s.get('name') == 'test_simple') + group_names = [g.get('name') for g in suite.findall('group')] + self.assertIn('physics', group_names) + + +# --------------------------------------------------------------------------- +# Test: subcycle suite +# --------------------------------------------------------------------------- + +class TestSubcycleIntegration(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_subcycle(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_group_cap_exists(self): + self.assertTrue( + os.path.isfile( + os.path.join(self._tmpdir, 'ccpp_test_subcycle_physics_cap.F90') + ) + ) + + def test_do_loop_in_group_cap(self): + with open( + os.path.join(self._tmpdir, 'ccpp_test_subcycle_physics_cap.F90') + ) as fh: + text = fh.read() + # ``suite_test_subcycle.xml`` declares ```` — + # the integer literal must flow through verbatim into the + # generated do-loop bound (no host-dict lookup, no symbolic + # translation). Compare against the literal value, not just the + # presence of any ``do ccpp_loop_counter = 1,``. + self.assertIn('do ccpp_loop_counter = 1, 3', text) + self.assertIn('end do', text) + + def test_datatable_for_subcycle_suite(self): + root = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')).getroot() + suites = root.find('api').find('suites') + names = [s.get('name') for s in suites.findall('suite')] + self.assertIn('test_subcycle', names) + + +# --------------------------------------------------------------------------- +# Test: multiple suites in one capgen run +# --------------------------------------------------------------------------- + +class TestMultipleSuites(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[_sf('scheme_multipart.meta')], + suite_files=[ + _suite_file('suite_test_simple.xml'), + _suite_file('suite_test_subcycle.xml'), + ], + output_root=self._tmpdir, + kind_types={}, + ) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_both_suite_caps_exist(self): + self.assertTrue( + os.path.isfile(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) + ) + self.assertTrue( + os.path.isfile(os.path.join(self._tmpdir, 'ccpp_test_subcycle_cap.F90')) + ) + + def test_host_cap_dispatches_both(self): + with open(os.path.join(self._tmpdir, 'test_host_ccpp_cap.F90')) as fh: + text = fh.read() + self.assertIn("case('test_simple')", text) + self.assertIn("case('test_subcycle')", text) + + def test_datatable_has_both_suites(self): + root = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')).getroot() + suites = root.find('api').find('suites') + names = [s.get('name') for s in suites.findall('suite')] + self.assertIn('test_simple', names) + self.assertIn('test_subcycle', names) + + def test_scheme_deduplicated_in_datatable(self): + root = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')).getroot() + scheme_names = [ + s.get('name') for s in root.find('schemes').findall('scheme') + ] + self.assertEqual(scheme_names.count('temp_calc_adjust'), 1) + + +# --------------------------------------------------------------------------- +# Test: multi-instance — number_of_instances flows end-to-end +# --------------------------------------------------------------------------- + +class TestMultiInstanceIntegration(unittest.TestCase): + """host_full.meta provides ninstances → generated code is multi-instance aware.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_ccpp_init_minimal_signature(self): + # Lifecycle signature for a multi-instance host carries the + # paired (inst_num, ninstances) control vars. + with open(os.path.join(self._tmpdir, 'test_host_ccpp_cap.F90')) as fh: + text = fh.read() + self.assertIn( + 'subroutine ccpp_init(suite_name, errflg, errmsg, inst_num, ninstances)', + text, + ) + + def test_suite_init_minimal_signature(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: + text = fh.read() + self.assertIn( + 'subroutine test_simple_init(inst_num, ninstances, errmsg, errflg)', + text, + ) + + def test_register_passes_ninstances_to_suite_state_alloc(self): + # _register USEs ninstances from the host module and passes it + # to the suite_state allocator (idempotent first-call alloc). + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: + text = fh.read() + self.assertIn( + 'call test_simple_suite_state_alloc(ninstances, errmsg, errflg)', + text, + ) + + def test_suite_init_passes_ninstances_to_group_state_alloc(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: + text = fh.read() + self.assertIn( + 'call physics_state_alloc(ninstances, errmsg, errflg)', + text, + ) + + def test_state_alloc_takes_number_of_instances_arg(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: + text = fh.read() + self.assertIn( + 'subroutine physics_state_alloc(number_of_instances, errmsg, errflg)', + text, + ) + + def test_state_alloc_uses_number_of_instances(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: + text = fh.read() + self.assertIn('allocate(ccpp_group_state(number_of_instances))', text) + + def test_group_state_alloc_is_idempotent(self): + """The generated group_state_alloc must short-circuit when the + array is already allocated — otherwise the second instance's + _init call crashes on a double-allocate. Mirrors the + ``_suite_state_alloc`` idempotency contract. + """ + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: + text = fh.read() + alloc_body = text.split('state_alloc(number_of_instances')[1] + alloc_body = alloc_body.split('end subroutine')[0] + self.assertIn('if (allocated(ccpp_group_state)) return', alloc_body) + # And the guard must precede the allocate, not follow it. + guard_pos = alloc_body.find('if (allocated(ccpp_group_state))') + alloc_pos = alloc_body.find('allocate(ccpp_group_state(') + self.assertGreater(alloc_pos, guard_pos, + "Idempotency guard must precede the allocate statement") + + def test_state_guard_uses_inst_num(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: + text = fh.read() + self.assertIn('ccpp_group_state(inst_num)', text) + + def test_group_init_has_inst_num_arg(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: + text = fh.read() + init_sub = text.split('subroutine physics_init')[1] + init_sub = init_sub.split('end subroutine')[0] + self.assertIn('inst_num', init_sub) + + def test_suite_cap_dispatches_inst_num_to_group_init(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: + text = fh.read() + physics_init = text.split('subroutine test_simple_physics_init')[1] + physics_init = physics_init.split('end subroutine')[0] + self.assertIn('inst_num', physics_init) + + +class TestSingleInstanceIntegration(unittest.TestCase): + """host_no_instance.meta + control_no_instance.meta omit the instance + pair → generated static API drops instance_number from public + signatures, suite cap uses literal '1' for state-array indexing, + state arrays are allocated with the literal '1' as bound. + """ + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + capgen( + host_name='test_host', + host_files=[ + _sf('host_no_instance.meta'), + _sf('control_no_instance.meta'), + ], + scheme_files=[_sf('scheme_multipart.meta')], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_host_cap_ccpp_init_omits_inst_num(self): + with open(os.path.join(self._tmpdir, 'test_host_ccpp_cap.F90')) as fh: + text = fh.read() + # ``suite_name_var`` is the local name declared by + # ``control_no_instance.meta`` for the suite_name control var. + self.assertIn( + 'subroutine ccpp_init(suite_name_var, errflg, errmsg)', + text, + ) + # And NOT the multi-instance shape. + self.assertNotIn('inst_num', text) + + def test_host_cap_ccpp_register_omits_inst_num(self): + with open(os.path.join(self._tmpdir, 'test_host_ccpp_cap.F90')) as fh: + text = fh.read() + self.assertIn( + 'subroutine ccpp_register(suite_name_var, errflg, errmsg)', + text, + ) + + def test_host_cap_ccpp_final_omits_inst_num(self): + with open(os.path.join(self._tmpdir, 'test_host_ccpp_cap.F90')) as fh: + text = fh.read() + self.assertIn( + 'subroutine ccpp_final(suite_name_var, errflg, errmsg)', + text, + ) + + def test_suite_init_omits_inst_num(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: + text = fh.read() + self.assertIn('subroutine test_simple_init(errmsg, errflg)', text) + + def test_register_passes_literal_one_to_state_alloc(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: + text = fh.read() + self.assertIn( + 'call test_simple_suite_state_alloc(1, errmsg, errflg)', text, + ) + + def test_suite_state_indexing_uses_literal_one(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: + text = fh.read() + self.assertIn('ccpp_suite_state(1)', text) + + def test_group_state_alloc_passes_literal_one(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: + text = fh.read() + self.assertIn( + 'call physics_state_alloc(1, errmsg, errflg)', + text, + ) + + def test_group_cap_init_omits_inst_num(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: + text = fh.read() + init_sub = text.split('subroutine physics_init')[1] + init_sub = init_sub.split('end subroutine')[0] + self.assertNotIn('inst_num', init_sub) + + +class TestInstancePairingErrors(unittest.TestCase): + """End-to-end: ``capgen`` rejects hosts that declare exactly one of the + instance_number / number_of_instances pair.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_instance_alone_raises(self): + from metadata.parse_tools import CCPPError + with self.assertRaises(CCPPError) as ctx: + capgen( + host_name='test_host', + host_files=[ + _sf('host_no_instance.meta'), + _sf('control_inst_only.meta'), # has instance_number only + ], + scheme_files=[_sf('scheme_multipart.meta')], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + msg = str(ctx.exception) + self.assertIn('instance_number', msg) + self.assertIn('number_of_instances', msg) + self.assertIn('paired', msg.lower()) + + def test_ninstances_alone_raises(self): + from metadata.parse_tools import CCPPError + with self.assertRaises(CCPPError) as ctx: + capgen( + host_name='test_host', + host_files=[ + _sf('host_no_instance.meta'), + _sf('control_ninst_only.meta'), # has ninstances only + ], + scheme_files=[_sf('scheme_multipart.meta')], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + msg = str(ctx.exception) + self.assertIn('instance_number', msg) + self.assertIn('number_of_instances', msg) + self.assertIn('paired', msg.lower()) + + +# --------------------------------------------------------------------------- +# Helper runners for ported test_prebuild test cases +# --------------------------------------------------------------------------- + +def _run_opt_arg(tmpdir): + capgen( + host_name='test_host', + host_files=[_sf('host_opt_arg.meta'), _sf('control_opt_arg.meta')], + scheme_files=[_sf('scheme_opt_arg.meta')], + suite_files=[_suite_file('suite_opt_arg.xml')], + output_root=tmpdir, + kind_types={}, + ) + return tmpdir + + +def _run_unit_conv(tmpdir): + capgen( + host_name='test_host', + host_files=[_sf('host_unit_conv.meta'), _sf('control_unit_conv.meta')], + scheme_files=[ + _sf('scheme_unit_conv_1.meta'), + _sf('scheme_unit_conv_2.meta'), + ], + suite_files=[_suite_file('suite_unit_conv.xml')], + output_root=tmpdir, + kind_types={}, + ) + return tmpdir + + +def _run_chunked_data(tmpdir): + capgen( + host_name='test_host', + host_files=[ + _sf('host_chunked_data.meta'), + _sf('ddt_chunked_data.meta'), + _sf('control_chunked_data.meta'), + ], + scheme_files=[_sf('scheme_chunked_data.meta')], + suite_files=[_suite_file('suite_chunked_data.xml')], + output_root=tmpdir, + kind_types={}, + ) + return tmpdir + + +# --------------------------------------------------------------------------- +# Test: suite types module (optional variable pointer wrappers) +# --------------------------------------------------------------------------- + +class TestSuiteTypesModule(unittest.TestCase): + """Suite types module is generated when optional args are present.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_opt_arg(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def _path(self, name): + return os.path.join(self._tmpdir, name) + + def test_types_module_exists(self): + self.assertTrue(os.path.isfile(self._path('ccpp_opt_arg_types.F90'))) + + def test_types_module_declaration(self): + with open(self._path('ccpp_opt_arg_types.F90')) as fh: + text = fh.read() + self.assertIn('module ccpp_opt_arg_types', text) + self.assertIn('end module ccpp_opt_arg_types', text) + + def test_integer_ptr_type_declared(self): + with open(self._path('ccpp_opt_arg_types.F90')) as fh: + text = fh.read() + self.assertIn('type :: integer_rank1_ptr_type', text) + self.assertIn('integer, pointer :: ptr(:) => null()', text) + + def test_real_kind_phys_ptr_type_declared(self): + with open(self._path('ccpp_opt_arg_types.F90')) as fh: + text = fh.read() + self.assertIn('type :: real_kind_phys_rank1_ptr_type', text) + self.assertIn('real(kind=kind_phys), pointer :: ptr(:) => null()', text) + + def test_types_module_public_declarations(self): + with open(self._path('ccpp_opt_arg_types.F90')) as fh: + text = fh.read() + self.assertIn('public :: integer_rank1_ptr_type', text) + self.assertIn('public :: real_kind_phys_rank1_ptr_type', text) + + def test_types_module_uses_ccpp_kinds(self): + with open(self._path('ccpp_opt_arg_types.F90')) as fh: + text = fh.read() + # The real_kind_phys_rank1_ptr_type uses kind_phys → must USE it. + self.assertIn('use ccpp_kinds, only: kind_phys', text) + + def test_no_types_module_for_simple_suite(self): + with tempfile.TemporaryDirectory() as d: + _run_simple(d) + self.assertFalse(os.path.isfile( + os.path.join(d, 'ccpp_test_simple_types.F90') + )) + + +# --------------------------------------------------------------------------- +# Test: optional variable handling (ported from test_prebuild/test_opt_arg) +# --------------------------------------------------------------------------- + +class TestOptArgIntegration(unittest.TestCase): + """End-to-end test with Case 2 (optional, no transform) and Case 4 + (optional + unit conversion km→m).""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_opt_arg(self._tmpdir) + with open( + os.path.join(self._tmpdir, 'ccpp_opt_arg_opt_arg_group_cap.F90') + ) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_group_cap_exists(self): + self.assertTrue( + os.path.isfile( + os.path.join(self._tmpdir, 'ccpp_opt_arg_opt_arg_group_cap.F90') + ) + ) + + def test_uses_types_module(self): + self.assertIn('use ccpp_opt_arg_types', self.text) + + def test_group_cap_uses_ccpp_kinds(self): + # Group cap declares ``real(kind=kind_phys)`` transformation locals + # → must USE kind_phys from ccpp_kinds. + self.assertIn('use ccpp_kinds, only: kind_phys', self.text) + + def test_case2_ptr_type_declared(self): + self.assertIn('type(integer_rank1_ptr_type) :: opt_var_p', self.text) + + def test_case4_ptr_and_temp_declared(self): + self.assertIn('type(real_kind_phys_rank1_ptr_type) :: opt_var_2_p', self.text) + self.assertIn('opt_var_2_l', self.text) + + def test_case4_temp_has_target_attr(self): + # The temp is the RHS of ``ptr%ptr => temp``, so it must be a TARGET. + # Match the declaration line: ``real(kind=kind_phys), dimension(nx), target :: opt_var_2_l`` + self.assertRegex( + self.text, + r'real\(kind=kind_phys\)[^\n]*,\s*target[^\n]*::\s*opt_var_2_l\b', + ) + + def test_case2_pre_call_active_guard(self): + self.assertIn('if (flag_for_opt_arg) then', self.text) + + def test_case2_ptr_assignment(self): + self.assertIn('opt_var_p%ptr => opt_arg', self.text) + + def test_case2_nullify_on_else(self): + self.assertIn('nullify(opt_var_p%ptr)', self.text) + + def test_case4_forward_unit_conversion(self): + # km → m: multiply by 1.0E+3 + self.assertIn('opt_var_2_l = 1.0E+3_kind_phys*opt_arg_2', self.text) + + def test_case4_ptr_to_temp(self): + self.assertIn('opt_var_2_p%ptr => opt_var_2_l', self.text) + + def test_case4_backward_unit_conversion(self): + # m → km: multiply by 1.0E-3 + self.assertIn('opt_arg_2', self.text) + self.assertIn('1.0E-3_kind_phys*opt_var_2_l', self.text) + + def test_optional_arg_passed_as_ptr(self): + self.assertIn('opt_var=opt_var_p%ptr', self.text) + self.assertIn('opt_var_2=opt_var_2_p%ptr', self.text) + + def test_active_condition_from_host(self): + # Active condition inherited from host metadata, not scheme metadata. + self.assertIn('flag_for_opt_arg', self.text) + + def test_types_in_datatable(self): + root = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')).getroot() + suite_files = root.find('capgen_files').find('suite_files') + names = [os.path.basename(f.text) for f in suite_files.findall('file')] + self.assertIn('ccpp_opt_arg_types.F90', names) + + +# --------------------------------------------------------------------------- +# Test: unit conversion (ported from test_prebuild/test_unit_conv, simplified) +# --------------------------------------------------------------------------- + +class TestSubcycleStdnameLoopBound(unittest.TestCase): + """End-to-end: a subcycle with ``loop=""`` must resolve to + the host's local Fortran name in the generated ``do`` loop, plus + emit the needed USE / dummy-arg threading.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + capgen( + host_name='test_host', + host_files=[ + _sf('host_full.meta'), + _sf('control_full.meta'), + _sf('host_subcycle_stdname.meta'), + ], + scheme_files=[_sf('scheme_multipart.meta')], + suite_files=[_suite_file('suite_subcycle_stdname.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + with open(os.path.join( + self._tmpdir, + 'ccpp_subcycle_stdname_suite_physics_cap.F90', + )) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_do_loop_uses_local_name(self): + """The do-loop bound is the host's local Fortran name (n_sub), + not the CCPP standard name (num_subcycles_for_test).""" + self.assertIn('do ccpp_loop_counter = 1, n_sub', self.text) + # And NOT the raw std name. + self.assertNotIn('do ccpp_loop_counter = 1, num_subcycles_for_test', + self.text) + + def test_use_statement_imports_loop_local(self): + """The host module exporting n_sub is USE'd so the symbol is + in scope inside the generated subroutine.""" + # n_sub comes from host_phys_subcycle_helper module. + self.assertIn('use host_phys_subcycle_helper', self.text) + self.assertIn('n_sub', self.text) + + +class TestHostTableDependenciesInDatatable(unittest.TestCase): + """A host table's ``dependencies =`` declarations must surface in + datatable.xml's section. The generator originally + only walked scheme_tables for deps, silently dropping host-table + declarations (real CCPP-physics hosts like SCM's GFS_typedefs.meta + declare many host-side file dependencies).""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + capgen( + host_name='test_host', + host_files=[ + _sf('host_with_dependencies.meta'), + _sf('control_full.meta'), + ], + scheme_files=[_sf('scheme_multipart.meta')], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + import xml.etree.ElementTree as ET + tree = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')) + self._deps = [ + d.text for d in tree.getroot() + .find('dependencies').findall('dependency') + ] + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_host_dependencies_listed(self): + """Every dep declared on the host table appears in + with the dependencies_path prefix resolved.""" + names = [os.path.basename(p) for p in self._deps] + self.assertIn('some_mp_params.F90', names) + self.assertIn('some_rad_param.f', names) + self.assertIn('some_chem.F90', names) + + def test_dependencies_path_applied_to_host_deps(self): + """The host's dependencies_path is resolved against each entry.""" + joined = '\n'.join(self._deps) + self.assertIn('/tmp/fake_phys/mp/some_mp_params.F90', joined) + self.assertIn('/tmp/fake_phys/radiation/some_rad_param.f', joined) + self.assertIn('/tmp/fake_phys/chemistry/some_chem.F90', joined) + + +class TestUnusedSchemeDependenciesFiltered(unittest.TestCase): + """Scheme metadata files supplied on the CLI but not referenced by + any loaded suite must not contribute to datatable.xml's + . Host build systems often pass the full physics + metadata catalog and rely on capgen to narrow the compile set. + """ + + _USED_DEP = '/tmp/used_phys/used_dep.F90' + _UNUSED_DEP = '/tmp/unused_phys/unused_dep.F90' + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + used = os.path.join(self._tmpdir, 'used_scheme.meta') + unused = os.path.join(self._tmpdir, 'unused_scheme.meta') + # The "used" scheme is the existing scheme_multipart fixture + # with a single ``dependencies =`` line spliced into its outer + # [ccpp-table-properties] block. + with open(_sf('scheme_multipart.meta')) as src: + body = src.read() + used_body = body.replace( + ' type = scheme\n', + ' type = scheme\n dependencies = {}\n'.format(self._USED_DEP), + 1, + ) + with open(used, 'w') as fh: + fh.write(used_body) + # The "unused" scheme has its own dependency. The SDF below + # references temp_calc_adjust only, so this file's deps must be + # filtered out of datatable.xml. + with open(unused, 'w') as fh: + fh.write( + "[ccpp-table-properties]\n" + " name = scheme_never_used\n" + " type = scheme\n" + " dependencies = {}\n" + "\n" + "[ccpp-arg-table]\n" + " name = scheme_never_used_run\n" + " type = scheme\n" + "[ errmsg ]\n" + " standard_name = ccpp_error_message\n" + " units = none\n" + " dimensions = ()\n" + " type = character\n" + " kind = len=512\n" + " intent = out\n" + "[ errflg ]\n" + " standard_name = ccpp_error_code\n" + " units = 1\n" + " dimensions = ()\n" + " type = integer\n" + " intent = out\n".format(self._UNUSED_DEP) + ) + # Drop a placeholder .F90 next to the used scheme's .meta so the + # source-path resolver picks it up rather than warning. The + # filename matches the .meta basename per the convention. + with open( + os.path.join(self._tmpdir, 'used_scheme.F90'), 'w' + ) as fh: + fh.write('! placeholder for source-path resolution\n') + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[used, unused], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + tree = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')) + root = tree.getroot() + self._deps = [ + d.text for d in root.find('dependencies').findall('dependency') + ] + self._scheme_files = [ + f.text for f in root.find('scheme_files').findall('file') + ] + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_used_scheme_dependency_present(self): + self.assertIn(self._USED_DEP, self._deps) + + def test_unused_scheme_dependency_absent(self): + self.assertNotIn(self._UNUSED_DEP, self._deps) + + def test_used_scheme_source_listed(self): + """ contains the resolved .F90 path for the used + scheme (same-base-name convention against the .meta file).""" + names = [os.path.basename(p) for p in self._scheme_files] + self.assertIn('used_scheme.F90', names) + + def test_unused_scheme_source_absent(self): + """A scheme metadata file passed on the CLI but not called by any + suite must not contribute its .F90 to .""" + names = [os.path.basename(p) for p in self._scheme_files] + self.assertNotIn('unused_scheme.F90', names) + self.assertNotIn('unused_scheme.f90', names) + + +class TestRegenerationIsNoopWhenContentUnchanged(unittest.TestCase): + """Running capgen twice with the same inputs must leave every + generated file's mtime untouched on the second invocation. This is + what ccpp-prebuild and original ccpp-capgen did so CMake / Make / + Ninja do NOT rebuild dependents on a no-op regeneration. The + generator stages each file via a sibling temp under the output root + and only replaces the target when content actually differs. + """ + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + self._first_run_files = sorted( + f for f in os.listdir(self._tmpdir) if not f.startswith('.') + ) + # Capture mtimes after the first run, then back-date everything + # by an hour so any rewrite is detectable as a bumped mtime. + self._old_mtimes = {} + old = time.time() - 3600 + for name in self._first_run_files: + path = os.path.join(self._tmpdir, name) + os.utime(path, (old, old)) + self._old_mtimes[name] = os.path.getmtime(path) + # Second run with identical inputs. + _run_simple(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_no_generated_file_was_rewritten(self): + unchanged = [] + rewritten = [] + for name, old in self._old_mtimes.items(): + path = os.path.join(self._tmpdir, name) + new = os.path.getmtime(path) + if new == old: + unchanged.append(name) + else: + rewritten.append((name, old, new)) + self.assertEqual( + rewritten, [], + "files rewritten on no-op regeneration: {}".format(rewritten), + ) + # Sanity check: we did see at least one file on disk to compare. + self.assertTrue(unchanged) + + def test_no_temp_artifacts_remain(self): + """The staging temp files (``.capgen_tmp_*``) must be cleaned up + after each run; none may survive into the next build step.""" + leftovers = [ + f for f in os.listdir(self._tmpdir) + if f.startswith('.capgen_tmp_') + ] + self.assertEqual(leftovers, []) + + +class TestRegenerationRewritesWhenContentChanges(unittest.TestCase): + """Negative of the above: when the inputs change between runs, the + affected generated files MUST be rewritten (bumped mtime). + """ + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + old = time.time() - 3600 + for name in os.listdir(self._tmpdir): + if name.startswith('.'): + continue + path = os.path.join(self._tmpdir, name) + os.utime(path, (old, old)) + # Second run picks a different SDF — exercises every cap file. + _run_simple(self._tmpdir, suite_xml='suite_test_subcycle.xml') + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_at_least_one_cap_was_rewritten(self): + """The subcycle suite differs from the simple suite — some cap + files MUST have a fresh mtime.""" + now = time.time() + any_recent = False + for name in os.listdir(self._tmpdir): + if name.startswith('.'): + continue + path = os.path.join(self._tmpdir, name) + if os.path.getmtime(path) > now - 60: + any_recent = True + break + self.assertTrue( + any_recent, + "no cap file was rewritten when the suite changed", + ) + + +class TestDdtDependenciesInSchemeMetaPreserved(unittest.TestCase): + """A scheme metadata file may carry a ``type = ddt`` block alongside + its ``type = scheme`` blocks (real-world pattern: a scheme that + constructs a DDT instance declares the DDT type in the same .meta). + The DDT block's ``dependencies = …`` must reach datatable.xml even + though the table name ('vmr_type', not the scheme name) won't match + the used-schemes set. Regression for the bug that broke the + end-to-end-tests/capgen test where ddt2.F90 went missing because + the DDT's deps were filtered out alongside actual scheme deps.""" + + _DDT_DEP = '/tmp/ddt_phys/inner_ddt.F90' + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + meta = os.path.join(self._tmpdir, 'mixed_scheme.meta') + # Splice a ``type = ddt`` table in front of the existing + # scheme_multipart body so the .meta carries both kinds. + with open(_sf('scheme_multipart.meta')) as src: + body = src.read() + ddt_block = ( + "[ccpp-table-properties]\n" + " name = inner_ddt_type\n" + " type = ddt\n" + " dependencies = {dep}\n" + "[ccpp-arg-table]\n" + " name = inner_ddt_type\n" + " type = ddt\n" + "[ pad ]\n" + " standard_name = inner_ddt_padding\n" + " units = count\n" + " dimensions = ()\n" + " type = integer\n" + "\n" + ).format(dep=self._DDT_DEP) + with open(meta, 'w') as fh: + fh.write(ddt_block) + fh.write(body) + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[meta], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + tree = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')) + self._deps = [ + d.text for d in tree.getroot() + .find('dependencies').findall('dependency') + ] + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_ddt_dependency_preserved(self): + self.assertIn(self._DDT_DEP, self._deps) + + +class TestSuiteInitFinalEmission(unittest.TestCase): + """End-to-end: an SDF with ```` and ```` at the suite + level produces calls to the named scheme's init/final phases inside + ``_init`` and ``_final``, with USE statements for the + scheme module and the standard error-flag check after the call.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[ + _sf('scheme_multipart.meta'), + _sf('scheme_suite_init_final.meta'), + ], + suite_files=[_suite_file('suite_with_init_final.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + with open(os.path.join( + self._tmpdir, 'ccpp_with_init_final_suite_cap.F90', + )) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_suite_init_calls_init_scheme(self): + sub = self.text.split('subroutine with_init_final_suite_init')[1] + sub = sub.split('end subroutine')[0] + self.assertIn('call suite_init_final_scheme_init', sub) + + def test_suite_final_calls_final_scheme(self): + sub = self.text.split('subroutine with_init_final_suite_final')[1] + sub = sub.split('end subroutine')[0] + self.assertIn('call suite_init_final_scheme_final', sub) + + def test_init_call_uses_scheme_module(self): + """The scheme module is USE'd inside ``_init`` so the + init subroutine is in scope.""" + sub = self.text.split('subroutine with_init_final_suite_init')[1] + sub = sub.split('end subroutine')[0] + self.assertIn( + 'use suite_init_final_scheme, only:', sub, + ) + self.assertIn('suite_init_final_scheme_init', sub) + + def test_final_call_uses_scheme_module(self): + sub = self.text.split('subroutine with_init_final_suite_final')[1] + sub = sub.split('end subroutine')[0] + self.assertIn( + 'use suite_init_final_scheme, only:', sub, + ) + self.assertIn('suite_init_final_scheme_final', sub) + + def test_init_call_precedes_state_transition(self): + """The init scheme is called BEFORE the FRAMEWORK_INITIALIZED + state transition — failures during the suite-init scheme stop + the state transition from firing.""" + sub = self.text.split('subroutine with_init_final_suite_init')[1] + sub = sub.split('end subroutine')[0] + call_pos = sub.index('call suite_init_final_scheme_init') + state_pos = sub.index('CCPP_SUITE_FRAMEWORK_INITIALIZED') + # There may be multiple state references (early-return guard); + # the final assignment is what matters. + state_set = sub.rindex( + 'ccpp_suite_state({}) = CCPP_SUITE_FRAMEWORK_INITIALIZED'.format( + 'inst_num' if 'inst_num' in sub else '1' + ) + ) + self.assertLess(call_pos, state_set) + + def test_final_call_precedes_unregister_transition(self): + sub = self.text.split('subroutine with_init_final_suite_final')[1] + sub = sub.split('end subroutine')[0] + call_pos = sub.index('call suite_init_final_scheme_final') + state_set = sub.index( + 'ccpp_suite_state({}) = CCPP_SUITE_UNREGISTERED'.format( + 'inst_num' if 'inst_num' in sub else '1' + ) + ) + self.assertLess(call_pos, state_set) + + def test_suite_init_final_scheme_listed_in_datatable(self): + """The suite-level / scheme is genuinely 'used', + so it must appear in datatable.xml's section alongside + any group-phase schemes. Without this, downstream consumers + (e.g. CMake glue iterating ) miss the file.""" + tree = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')) + names = { + s.get('name') + for s in tree.getroot().find('schemes').findall('scheme') + } + self.assertIn('suite_init_final_scheme', names) + self.assertIn('temp_calc_adjust', names) + + +class TestNestedSubcycleEmission(unittest.TestCase): + """End-to-end: a nested ```` in the SDF must produce + nested ``do`` loops in the generated cap. Without this, schemes + inside the inner loops run fewer times than the SDF specified, + silently producing wrong numerical answers.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[_sf('scheme_multipart.meta')], + suite_files=[_suite_file('suite_nested_subcycle.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + with open(os.path.join( + self._tmpdir, + 'ccpp_nested_subcycle_suite_physics_cap.F90', + )) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_outer_loop_present(self): + """The outer loop uses ``ccpp_loop_counter`` (preserves the + existing single-level convention).""" + self.assertIn('do ccpp_loop_counter = 1, 3', self.text) + + def test_inner_loop_present(self): + """The inner loop uses ``ccpp_loop_counter_2`` so the two loop + vars are distinct in the same scope (Fortran requires it).""" + self.assertIn('do ccpp_loop_counter_2 = 1, 2', self.text) + + def test_two_end_do_statements(self): + """Each nesting level closes with an ``end do``.""" + # Find the run-phase body to scope the check. + run = self.text.split('subroutine physics_run')[1] + run = run.split('end subroutine')[0] + self.assertEqual(run.count('end do'), 2) + + def test_inner_loop_inside_outer(self): + """The inner ``do`` line appears AFTER the outer ``do`` line + and BEFORE the first ``end do``.""" + run = self.text.split('subroutine physics_run')[1] + run = run.split('end subroutine')[0] + outer = run.index('do ccpp_loop_counter = 1, 3') + inner = run.index('do ccpp_loop_counter_2 = 1, 2') + first_end = run.index('end do') + self.assertLess(outer, inner) + self.assertLess(inner, first_end) + + def test_scheme_call_inside_inner_loop(self): + """The scheme call is nested inside the inner loop — not in + between the loops.""" + run = self.text.split('subroutine physics_run')[1] + run = run.split('end subroutine')[0] + inner = run.index('do ccpp_loop_counter_2 = 1, 2') + call = run.index('call temp_calc_adjust_run') + # First end do is the inner loop's close. + first_end = run.index('end do') + self.assertLess(inner, call) + self.assertLess(call, first_end) + + def test_two_counter_declarations(self): + """Each loop variable has its own integer declaration.""" + self.assertIn('integer :: ccpp_loop_counter\n', self.text) + self.assertIn('integer :: ccpp_loop_counter_2\n', self.text) + + +class TestSubcycleStdnameLoopBoundDdt(unittest.TestCase): + """End-to-end: a subcycle ``loop=""`` that resolves to a + DDT-component must emit the access path (``%``) as the + loop bound and USE the DDT instance's parent module by the *root* + of that access path — not the bare component name. + """ + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + capgen( + host_name='test_host', + host_files=[ + _sf('host_full.meta'), + _sf('control_full.meta'), + _sf('host_subcycle_stdname_ddt.meta'), + _sf('ddt_subcycle_stdname.meta'), + ], + scheme_files=[_sf('scheme_multipart.meta')], + suite_files=[_suite_file('suite_subcycle_stdname_ddt.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + with open(os.path.join( + self._tmpdir, + 'ccpp_subcycle_stdname_ddt_suite_physics_cap.F90', + )) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_do_loop_uses_access_path(self): + """The do-loop bound is the access path of the DDT component, + not the bare component name.""" + self.assertIn( + 'do ccpp_loop_counter = 1, phys_state%n_sub', self.text, + ) + # And NOT the bare component or the standard name. + self.assertNotIn('do ccpp_loop_counter = 1, n_sub\n', self.text) + self.assertNotIn( + 'do ccpp_loop_counter = 1, num_subcycles_for_test', self.text, + ) + + def test_use_imports_ddt_instance_root(self): + """The USE statement targets the DDT *instance* (phys_state), + not the bare component (n_sub).""" + # Some line starting with `use test_host_with_ddt_mod, only: ...` + # must list phys_state — n_sub is not a free module symbol. + self.assertIn('phys_state', self.text) + # The bare component must not be on the ``only:`` clause. + import re + m = re.search( + r'use\s+test_host_with_ddt_mod\s*,\s*only:\s*([^\n]+)', + self.text, + ) + self.assertIsNotNone(m) + only_clause = m.group(1) + self.assertIn('phys_state', only_clause) + # Word-boundary check: ``n_sub`` may legitimately appear inside + # ``phys_state%n_sub`` further down in the file, but it must not + # be a free symbol on this USE line. + symbols = [s.strip() for s in only_clause.split(',')] + self.assertNotIn('n_sub', symbols) + + +class TestModuleNameOverrideIntegration(unittest.TestCase): + """End-to-end: a scheme whose ``[ccpp-table-properties]`` declares an + explicit ``module_name`` must cause the generated cap to emit + ``use `` rather than the table's ``name``.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[_sf('scheme_module_name_override.meta')], + suite_files=[_suite_file('suite_module_name_override.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + with open(os.path.join( + self._tmpdir, + 'ccpp_module_name_override_suite_physics_cap.F90', + )) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_group_cap_uses_module_name_not_scheme_name(self): + """The USE statement targets the Fortran module name.""" + self.assertIn('use mod_alt_name, only:', self.text) + + def test_group_cap_does_not_use_scheme_name_directly(self): + """The scheme-name token (``scheme_alt_name``) must not appear as a + module on a ``use`` line — it would resolve to a non-existent + module file at compile time.""" + # Word-boundary check: ``scheme_alt_name_run`` (the subroutine) is + # legitimately mentioned in the ``only:`` clause. + import re + bad = re.search(r'use\s+scheme_alt_name\s*,', self.text) + self.assertIsNone(bad, + "Cap should not emit 'use scheme_alt_name, ...' when the " + "metadata declares module_name = mod_alt_name") + + def test_only_clause_carries_phase_subroutine(self): + """The ``only:`` clause exposes ``_`` symbols + from the renamed module — Fortran subroutine names stay tied to + the scheme name, not the module name.""" + self.assertIn('scheme_alt_name_run', self.text) + + +class TestVerticalFlipIntegration(unittest.TestCase): + """End-to-end: host declares air_temperature with default + top_at_one = False; scheme declares top_at_one = True. The + generated group cap must emit a temp + flipped subscript on the + host-side access expression and pass the temp to the scheme. + """ + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[_sf('scheme_top_at_one.meta')], + suite_files=[_suite_file('suite_top_at_one.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + with open( + os.path.join( + self._tmpdir, + 'ccpp_top_at_one_suite_physics_cap.F90', + ) + ) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_group_cap_exists(self): + self.assertTrue( + os.path.isfile( + os.path.join( + self._tmpdir, + 'ccpp_top_at_one_suite_physics_cap.F90', + ) + ) + ) + + def test_temp_local_declared(self): + # Transform pipeline kicks in: temp local named ``temp_l`` with + # scheme dimensions (lb:ub on horizontal, nlev on vertical — the + # vertical bound is single-extent shorthand for 1:nlev). + self.assertIn('real(kind=kind_phys), dimension(lb:ub, nlev) :: temp_l', + self.text) + + def test_pre_call_forward_uses_flipped_subscript(self): + # Host metadata declares air_temperature as ``gt0`` with default + # top_at_one = False; scheme wants top_at_one = True → reverse + # stride on the vertical axis of the host-side expression. + self.assertIn('temp_l = gt0(lb:ub, nlev:1:-1)', self.text) + + def test_post_call_backward_writes_into_flipped_lhs(self): + # inout, so backward also fires; writes temp back into host with + # the flipped subscript as LHS. + self.assertIn('gt0(lb:ub, nlev:1:-1) = temp_l', self.text) + + def test_call_site_passes_temp_not_host(self): + # The scheme is called with the temp local, not the host expression. + call_block = self.text.split('call top_at_one_scheme_run')[1] + call_block = call_block.split(')')[0] + self.assertIn('temp=temp_l', call_block) + + def test_comment_mentions_vertical_flip(self): + self.assertIn('vertical flip', self.text) + + +class TestUnitConvIntegration(unittest.TestCase): + """Two schemes in one group: scheme_1 needs m (Case 1/2), + scheme_2 needs km (Case 3/4 — m→km forward, km→m backward).""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_unit_conv(self._tmpdir) + with open( + os.path.join( + self._tmpdir, 'ccpp_unit_conv_unit_conv_group_cap.F90' + ) + ) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_group_cap_exists(self): + self.assertTrue( + os.path.isfile( + os.path.join( + self._tmpdir, 'ccpp_unit_conv_unit_conv_group_cap.F90' + ) + ) + ) + + def test_case1_direct_call_scheme_1(self): + # Scheme 1 wants m, host has m — direct reference, no temp. + self.assertIn('call unit_conv_scheme_1_run', self.text) + # data_array should be passed directly (no _l suffix for scheme_1 data_array) + self.assertIn('data_array=data_array(lb:ub)', self.text) + + def test_case3_temp_forward_for_scheme_2(self): + # Scheme 2 wants km: m→km conversion. + self.assertIn('1.0E-3_kind_phys*data_array(lb:ub)', self.text) + + def test_case3_backward_for_scheme_2(self): + # km→m backward conversion. + self.assertIn('1.0E+3_kind_phys*', self.text) + + def test_case2_optional_ptr_for_scheme_1(self): + # Scheme 1 optional data_array_opt: same units, pointer-only. + self.assertIn('data_array_opt_p', self.text) + + def test_case4_optional_plus_transform_for_scheme_2(self): + # Scheme 2 optional data_array_opt: pointer + km transform. + self.assertIn('data_array_opt_2_p', self.text) + + def test_types_module_for_unit_conv(self): + self.assertTrue( + os.path.isfile( + os.path.join(self._tmpdir, 'ccpp_unit_conv_types.F90') + ) + ) + + +# --------------------------------------------------------------------------- +# Test: chunked data (ported from test_prebuild/test_chunked_data) +# --------------------------------------------------------------------------- + +class TestChunkedDataIntegration(unittest.TestCase): + """Suite with a DDT-based host variable accessed by a scheme.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_chunked_data(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_group_cap_exists(self): + self.assertTrue( + os.path.isfile( + os.path.join( + self._tmpdir, + 'ccpp_chunked_data_chunked_data_group_cap.F90', + ) + ) + ) + + def test_scheme_run_calls_present(self): + with open( + os.path.join( + self._tmpdir, + 'ccpp_chunked_data_chunked_data_group_cap.F90', + ) + ) as fh: + text = fh.read() + self.assertIn('call chunked_data_scheme_run', text) + + def test_ddt_access_in_group_cap(self): + with open( + os.path.join( + self._tmpdir, + 'ccpp_chunked_data_chunked_data_group_cap.F90', + ) + ) as fh: + text = fh.read() + # DDT field access should appear in the call expression. + self.assertIn('chunked_data_instance%array_data', text) + + def test_no_types_module_for_chunked_data(self): + self.assertFalse( + os.path.isfile( + os.path.join(self._tmpdir, 'ccpp_chunked_data_types.F90') + ) + ) + + def test_scheme_module_imported(self): + with open( + os.path.join( + self._tmpdir, + 'ccpp_chunked_data_chunked_data_group_cap.F90', + ) + ) as fh: + text = fh.read() + # Group cap must import the actual scheme module so that + # `call chunked_data_scheme_(...)` resolves. + self.assertIn('use chunked_data_scheme, only:', text) + for phase in ('init', 'timestep_init', 'run', + 'timestep_final', 'final'): + self.assertIn('chunked_data_scheme_{}'.format(phase), text) + + def test_phase_subroutine_order_canonical(self): + with open( + os.path.join( + self._tmpdir, + 'ccpp_chunked_data_chunked_data_group_cap.F90', + ) + ) as fh: + text = fh.read() + # Phase subroutines and public declarations must appear in canonical + # order: init, timestep_init, run, timestep_final, final, then + # state_alloc, state_dealloc. + canonical = [ + '_init', '_timestep_init', '_run', '_timestep_final', '_final', + '_state_alloc', '_state_dealloc', + ] + positions = [ + text.index('public :: chunked_data_group{}'.format(s)) + for s in canonical + ] + self.assertEqual(positions, sorted(positions)) + + def test_datatable_has_chunked_data_suite(self): + root = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')).getroot() + suites = root.find('api').find('suites') + names = [s.get('name') for s in suites.findall('suite')] + self.assertIn('chunked_data', names) + + +# --------------------------------------------------------------------------- +# Test: interstitial suite-owned data — allocation and multi-instance +# --------------------------------------------------------------------------- + +def _run_interstitial(tmpdir): + from ccpp_capgen import capgen + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[ + _sf('scheme_interstitial_producer.meta'), + _sf('scheme_interstitial_consumer.meta'), + ], + suite_files=[_suite_file('suite_interstitial.xml')], + output_root=tmpdir, + kind_types={}, + ) + + +class TestInterstitialSuiteData(unittest.TestCase): + """Suite-owned interstitial variable: deferred-shape DDT, allocatable + instance array, suite_state_alloc wired into suite init.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_interstitial(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def _data(self): + return open(os.path.join(self._tmpdir, 'ccpp_interstitial_data.F90')).read() + + def _suite_cap(self): + return open(os.path.join(self._tmpdir, 'ccpp_interstitial_cap.F90')).read() + + def _group_cap(self): + return open(os.path.join(self._tmpdir, + 'ccpp_interstitial_diag_group_cap.F90')).read() + + def test_suite_data_field_uses_deferred_shape(self): + """Allocatable field must use (:,:) not dimension(std_name,...).""" + text = self._data() + self.assertIn('(:,:)', text) + self.assertNotIn('dimension(horizontal_dimension', text) + + def test_suite_data_has_allocatable_instance_array(self): + text = self._data() + self.assertIn('type(ccpp_interstitial_data_t), allocatable', text) + self.assertIn('ccpp_suite_data(:)', text) + + def test_suite_data_alloc_subroutine_exists(self): + self.assertIn('suite_data_alloc', self._data()) + + def test_suite_data_init_fields_uses_host_dims(self): + # init_fields now owns the inner allocations; it needs the host dims. + text = self._data() + init_fields = text.split('subroutine suite_data_init_fields')[1].split('end subroutine')[0] + self.assertIn('use host_phys', init_fields) + self.assertIn('ncols', init_fields) + self.assertIn('nlev', init_fields) + + def test_suite_data_alloc_allocates_outer_array(self): + self.assertIn('allocate(ccpp_suite_data(number_of_instances))', self._data()) + + def test_suite_data_init_fields_allocates_field_per_instance(self): + # Inner allocations moved out of suite_data_alloc into init_fields so + # suite-owned dims (e.g. set during _register) can be picked up after + # the register phase has run. + text = self._data() + init_fields = text.split('subroutine suite_data_init_fields')[1].split('end subroutine')[0] + self.assertIn('allocate(ccpp_suite_data(i)%diag_out(ncols, nlev))', init_fields) + + def test_suite_data_dealloc_subroutine_exists(self): + self.assertIn('suite_data_dealloc', self._data()) + + def test_suite_data_final_fields_subroutine_exists(self): + self.assertIn('suite_data_final_fields', self._data()) + + def test_suite_cap_has_suite_state_alloc(self): + self.assertIn('interstitial_suite_state_alloc', self._suite_cap()) + + def test_suite_cap_register_calls_suite_state_alloc(self): + # State + DDT-array allocation has moved from _init into + # _register so register can be the first lifecycle entry point. + text = self._suite_cap() + register_body = text.split('subroutine interstitial_register')[1].split('end subroutine')[0] + self.assertIn('interstitial_suite_state_alloc', register_body) + + def test_suite_state_alloc_calls_suite_data_alloc(self): + text = self._suite_cap() + alloc_body = text.split('subroutine interstitial_suite_state_alloc')[1].split('end subroutine')[0] + self.assertIn('suite_data_alloc', alloc_body) + + def test_suite_init_calls_init_fields(self): + # _init triggers per-instance inner allocations. + text = self._suite_cap() + init_body = text.split('subroutine interstitial_init')[1].split('end subroutine')[0] + self.assertIn('suite_data_init_fields', init_body) + + def test_suite_state_alloc_allocates_state_array(self): + self.assertIn('allocate(ccpp_suite_state(number_of_instances))', self._suite_cap()) + + def test_suite_cap_final_calls_suite_state_dealloc(self): + text = self._suite_cap() + final_body = text.split('subroutine interstitial_final')[1].split('end subroutine')[0] + self.assertIn('interstitial_suite_state_dealloc', final_body) + + def test_group_cap_uses_suite_data_module(self): + self.assertIn('use ccpp_interstitial_data', self._group_cap()) + + def test_group_cap_accesses_suite_data_with_instance(self): + """Suite var access must index ccpp_suite_data by instance.""" + text = self._group_cap() + self.assertIn('ccpp_suite_data(', text) + + def test_group_run_has_inst_num_dummy_arg(self): + """Gap 3: instance_number must be a dummy arg when suite vars are referenced.""" + text = self._group_cap() + run_sub = text.split('subroutine diag_group_run')[1] + run_sub = run_sub.split('end subroutine')[0] + self.assertIn('inst_num', run_sub) + self.assertIn('integer, intent(in)', run_sub) + + def test_suite_cap_passes_inst_num_to_group_run(self): + """Suite cap physics_run dispatch must pass inst_num to group run.""" + text = self._suite_cap() + physics_run = text.split('subroutine interstitial_physics_run')[1] + physics_run = physics_run.split('end subroutine')[0] + self.assertIn('inst_num', physics_run) + self.assertIn('call diag_group_run', physics_run) + + def test_host_cap_physics_run_has_inst_num(self): + """Static API ccpp_physics_run must include inst_num when groups need it.""" + with open(os.path.join(self._tmpdir, 'test_host_ccpp_cap.F90')) as fh: + text = fh.read() + physics_run = text.split('subroutine ccpp_physics_run')[1] + physics_run = physics_run.split('end subroutine')[0] + self.assertIn('inst_num', physics_run) + + def test_suite_meta_file_exists(self): + """Polish 2: ccpp__data.meta must be generated.""" + self.assertTrue( + os.path.isfile(os.path.join(self._tmpdir, 'ccpp_interstitial_data.meta')) + ) + + def test_suite_meta_contains_suite_var(self): + """Suite meta file must list suite-owned variable.""" + with open(os.path.join(self._tmpdir, 'ccpp_interstitial_data.meta')) as fh: + text = fh.read() + self.assertIn('diagnostic_interstitial_field', text) + self.assertIn('type = suite', text) + self.assertIn('standard_name = diagnostic_interstitial_field', text) + + +# --------------------------------------------------------------------------- +# Test: Gap 1 — active expression variables in USE statements +# --------------------------------------------------------------------------- + +class TestActiveVarInUseStatements(unittest.TestCase): + """Gap 1: Variables referenced in active= expressions must appear in USE.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + from test_integration import _run_unit_conv + _run_unit_conv(self._tmpdir) + with open( + os.path.join(self._tmpdir, 'ccpp_unit_conv_unit_conv_group_cap.F90') + ) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_active_flag_in_use_statement(self): + """flag_for_opt_array is referenced in active= but was not a direct arg.""" + self.assertIn('flag_for_opt_array', self.text.split('contains')[0]) + + def test_active_flag_is_use_d_not_declared(self): + """The active flag must appear in a use statement, not a dummy declaration.""" + use_section = self.text.split('implicit none')[0] + self.assertIn('flag_for_opt_array', use_section) + + +# --------------------------------------------------------------------------- +# Test: Gap 2 — temp variable declarations use local names not standard names +# --------------------------------------------------------------------------- + +class TestTempDeclLocalNames(unittest.TestCase): + """Gap 2: Temp locals for unit conversion must be declared with local names.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + from test_integration import _run_unit_conv + _run_unit_conv(self._tmpdir) + with open( + os.path.join(self._tmpdir, 'ccpp_unit_conv_unit_conv_group_cap.F90') + ) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_temp_uses_chunk_bounds(self): + """Temp array for a horizontal_dimension scheme arg must be sized + with the chunk loop bounds (host's local names for + horizontal_loop_begin / horizontal_loop_end), NOT the full-extent + name. Using ``dimension(ncols)`` over-sizes the temp and produces + a Fortran shape mismatch on the unit-conversion assignment like + ``data_array_l = factor * data_array(lb:ub)``. + + The unit_conv fixture's control table uses ``lb`` / ``ub`` as the + local names for horizontal_loop_begin / horizontal_loop_end. + """ + self.assertIn('dimension(lb:ub)', self.text) + # And NOT the full-extent name. + self.assertNotIn('dimension(ncols)', self.text) + + def test_temp_does_not_use_standard_name(self): + """Standard name must not appear in a dimension() declaration.""" + self.assertNotIn('dimension(horizontal_dimension)', self.text) + self.assertNotIn('dimension(horizontal_loop_extent)', self.text) + self.assertNotIn('dimension(horizontal_loop_begin', self.text) + + +class TestTempDeclLocalNamesOptArg(unittest.TestCase): + """Gap 2: Temp decl uses nx (local) not size_of_std_arg (standard name).""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + from test_integration import _run_opt_arg + _run_opt_arg(self._tmpdir) + with open( + os.path.join(self._tmpdir, 'ccpp_opt_arg_opt_arg_group_cap.F90') + ) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_temp_uses_local_dim_name(self): + self.assertIn('dimension(nx)', self.text) + + def test_temp_does_not_use_standard_name(self): + self.assertNotIn('dimension(size_of_std_arg)', self.text) + + +# --------------------------------------------------------------------------- +# Test: Polish 1 — timestep_init/timestep_final phase state guards +# --------------------------------------------------------------------------- + +class TestTimestepStateGuards(unittest.TestCase): + """Polish 1: timestep_init must guard on IN_TIMESTEP; timestep_final resets.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + from test_integration import _run_opt_arg + _run_opt_arg(self._tmpdir) + with open( + os.path.join(self._tmpdir, 'ccpp_opt_arg_opt_arg_group_cap.F90') + ) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_timestep_init_entry_guard(self): + """timestep_init subroutine must have IN_TIMESTEP entry guard.""" + ts_init = self.text.split( + 'subroutine opt_arg_group_timestep_init' + )[1].split('end subroutine')[0] + self.assertIn('CCPP_GROUP_IN_TIMESTEP', ts_init) + self.assertIn('return', ts_init) + + def test_timestep_init_sets_in_timestep(self): + """timestep_init must set state to IN_TIMESTEP after scheme calls.""" + ts_init = self.text.split( + 'subroutine opt_arg_group_timestep_init' + )[1].split('end subroutine')[0] + self.assertIn('ccpp_group_state', ts_init) + self.assertIn('CCPP_GROUP_IN_TIMESTEP', ts_init) + + def test_timestep_final_resets_to_initialized(self): + """timestep_final must reset state to INITIALIZED.""" + ts_final = self.text.split( + 'subroutine opt_arg_group_timestep_final' + )[1].split('end subroutine')[0] + self.assertIn('CCPP_GROUP_INITIALIZED', ts_final) + self.assertIn('ccpp_group_state', ts_final) + + +# --------------------------------------------------------------------------- +# Test: Polish 2 — ccpp__data.meta output file +# --------------------------------------------------------------------------- + +class TestSuiteMetaOutput(unittest.TestCase): + """Polish 2: ccpp__data.meta must be written for every suite.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_meta_file_created(self): + self.assertTrue( + os.path.isfile(os.path.join(self._tmpdir, 'ccpp_test_simple_data.meta')) + ) + + def test_meta_header_comment(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_data.meta')) as fh: + text = fh.read() + self.assertTrue(text.startswith('!')) + self.assertIn('ccpp_test_simple', text.split('\n')[0]) + + def test_meta_table_properties(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_data.meta')) as fh: + text = fh.read() + self.assertIn('[ccpp-table-properties]', text) + self.assertIn('name = ccpp_test_simple_data', text) + self.assertIn('type = suite', text) + + def test_meta_empty_suite_has_no_var_entries(self): + """simple suite has no suite-owned vars so no [ var ] blocks.""" + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_data.meta')) as fh: + text = fh.read() + self.assertNotIn('standard_name =', text) + + def test_meta_with_suite_vars_lists_them(self): + """Interstitial suite's meta must list the interstitial variable.""" + with tempfile.TemporaryDirectory() as d: + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[ + _sf('scheme_interstitial_producer.meta'), + _sf('scheme_interstitial_consumer.meta'), + ], + suite_files=[_suite_file('suite_interstitial.xml')], + output_root=d, + kind_types={}, + ) + with open(os.path.join(d, 'ccpp_interstitial_data.meta')) as fh: + text = fh.read() + self.assertIn('standard_name = diagnostic_interstitial_field', text) + self.assertIn('units = K', text) + self.assertIn('type = real', text) + self.assertIn('kind = kind_phys', text) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_io_helpers.py b/unit-tests/test_io_helpers.py new file mode 100644 index 00000000..ecb4c53c --- /dev/null +++ b/unit-tests/test_io_helpers.py @@ -0,0 +1,189 @@ +"""Unit tests for :mod:`metadata.parse_tools.io_helpers`. + +These cover the write-if-changed contract: + * unchanged content leaves the on-disk mtime untouched, so downstream + build tools (CMake, Make, Ninja) do not trigger unnecessary + recompiles; + * changed content (or missing file) results in an atomic + same-directory replace; + * the temp file lives under the target's parent directory (which sits + under the generator's output root), never ``/tmp``. +""" + +import logging +import os +import tempfile +import time +import unittest + +from metadata.parse_tools.io_helpers import open_if_changed, write_if_changed + + +class _Capture(logging.Handler): + """Tiny in-memory log handler for assertions.""" + + def __init__(self): + super().__init__(level=logging.DEBUG) + self.records = [] + + def emit(self, record): + self.records.append(self.format(record)) + + +class TestWriteIfChanged(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + self._path = os.path.join(self._tmpdir, 'out.txt') + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_creates_file_when_missing(self): + self.assertFalse(os.path.exists(self._path)) + wrote = write_if_changed(self._path, 'hello\n') + self.assertTrue(wrote) + with open(self._path) as fh: + self.assertEqual(fh.read(), 'hello\n') + + def test_skips_write_when_content_identical(self): + write_if_changed(self._path, 'hello\n') + # Set an old mtime so we can detect any rewrite. + old_mtime = time.time() - 3600 + os.utime(self._path, (old_mtime, old_mtime)) + observed = os.path.getmtime(self._path) + + wrote = write_if_changed(self._path, 'hello\n') + + self.assertFalse(wrote) + self.assertEqual(os.path.getmtime(self._path), observed) + + def test_rewrites_when_content_differs(self): + write_if_changed(self._path, 'hello\n') + old_mtime = time.time() - 3600 + os.utime(self._path, (old_mtime, old_mtime)) + + wrote = write_if_changed(self._path, 'goodbye\n') + + self.assertTrue(wrote) + self.assertGreater(os.path.getmtime(self._path), old_mtime) + with open(self._path) as fh: + self.assertEqual(fh.read(), 'goodbye\n') + + def test_temp_file_lives_under_target_directory(self): + """The staging temp file must be under the target's parent dir + (which is under output_root), not /tmp. Verified by listing + siblings of the target during a write.""" + observed_siblings = [] + + # Patch tempfile.mkstemp via a wrapper: we can't intercept inside + # the helper, but we can verify the contract by writing many + # files in the same dir and asserting no .capgen_tmp_* survives. + for i in range(5): + write_if_changed(self._path, 'iteration_{}\n'.format(i)) + observed_siblings.append(set(os.listdir(self._tmpdir))) + + for snapshot in observed_siblings: + for name in snapshot: + self.assertFalse( + name.startswith('.capgen_tmp_'), + "leaked temp file: {}".format(name), + ) + + def test_creates_parent_directory_if_missing(self): + nested = os.path.join(self._tmpdir, 'a', 'b', 'c', 'out.txt') + write_if_changed(nested, 'hi\n') + self.assertTrue(os.path.isfile(nested)) + + +class TestOpenIfChanged(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + self._path = os.path.join(self._tmpdir, 'out.txt') + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_drop_in_for_open_write(self): + with open_if_changed(self._path) as fh: + fh.write('line1\n') + fh.write('line2\n') + with open(self._path) as fh: + self.assertEqual(fh.read(), 'line1\nline2\n') + + def test_idempotent_no_mtime_bump(self): + with open_if_changed(self._path) as fh: + fh.write('stable\n') + old_mtime = time.time() - 3600 + os.utime(self._path, (old_mtime, old_mtime)) + + with open_if_changed(self._path) as fh: + fh.write('stable\n') + + self.assertEqual(os.path.getmtime(self._path), old_mtime) + + def test_rejects_non_write_mode(self): + with self.assertRaises(ValueError): + with open_if_changed(self._path, mode='r'): + pass + + +class TestLogging(unittest.TestCase): + """The helper emits one info-level log line per call when *logger* is + supplied — ``Wrote `` on write, ``Unchanged: `` on no-op. + Build-tool users rely on the wording to tell at a glance which + generated files actually changed on a rerun.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + self._path = os.path.join(self._tmpdir, 'out.txt') + self._logger = logging.getLogger('test_io_helpers') + self._logger.setLevel(logging.DEBUG) + # Strip any preexisting handlers so we only see ours. + for h in list(self._logger.handlers): + self._logger.removeHandler(h) + self._handler = _Capture() + self._handler.setFormatter(logging.Formatter('%(levelname)s %(message)s')) + self._logger.addHandler(self._handler) + self._logger.propagate = False + + def tearDown(self): + self._logger.removeHandler(self._handler) + import shutil + shutil.rmtree(self._tmpdir) + + def test_logs_wrote_on_new_file(self): + write_if_changed(self._path, 'hi\n', logger=self._logger) + joined = '\n'.join(self._handler.records) + self.assertIn('INFO Wrote', joined) + self.assertIn(self._path, joined) + self.assertNotIn('Unchanged', joined) + + def test_logs_unchanged_on_identical_rewrite(self): + write_if_changed(self._path, 'hi\n', logger=self._logger) + self._handler.records.clear() + write_if_changed(self._path, 'hi\n', logger=self._logger) + joined = '\n'.join(self._handler.records) + self.assertIn('INFO Unchanged:', joined) + self.assertIn(self._path, joined) + self.assertNotIn('Wrote', joined) + + def test_logs_wrote_on_content_change(self): + write_if_changed(self._path, 'hi\n', logger=self._logger) + self._handler.records.clear() + write_if_changed(self._path, 'bye\n', logger=self._logger) + joined = '\n'.join(self._handler.records) + self.assertIn('INFO Wrote', joined) + self.assertNotIn('Unchanged', joined) + + def test_open_if_changed_forwards_logger(self): + with open_if_changed(self._path, logger=self._logger) as fh: + fh.write('hi\n') + self.assertIn('INFO Wrote', '\n'.join(self._handler.records)) + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/test_kinds_writer.py b/unit-tests/test_kinds_writer.py new file mode 100644 index 00000000..d15baf5e --- /dev/null +++ b/unit-tests/test_kinds_writer.py @@ -0,0 +1,156 @@ +"""Unit tests for generator.kinds_writer.""" + +import doctest +import os +import tempfile +import unittest + +from metadata.parse_tools import CCPPError +from generator.kinds_writer import _generate_ccpp_kinds, write_ccpp_kinds + +_ISO = 'iso_fortran_env' + + +class TestGenerateCcppKinds(unittest.TestCase): + + def test_single_kind(self): + lines = _generate_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}) + self.assertIn('module ccpp_kinds', lines) + self.assertIn('end module ccpp_kinds', lines) + self.assertTrue(any('kind_phys' in l and 'REAL64' in l for l in lines)) + self.assertTrue(any('parameter' in l and 'public' in l for l in lines)) + + def test_iso_use_single(self): + lines = _generate_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}) + iso_line = next(l for l in lines if 'iso_fortran_env' in l) + self.assertIn('REAL64', iso_line) + self.assertNotIn('&', iso_line) + + def test_iso_use_shared_spec_deduped(self): + """Two kinds sharing the same spec → spec listed once.""" + lines = _generate_ccpp_kinds({ + 'kind_a': (_ISO, 'REAL64'), + 'kind_b': (_ISO, 'REAL64'), + }) + iso_lines = [l for l in lines if 'iso_fortran_env' in l] + iso_text = ' '.join(iso_lines) + self.assertEqual(iso_text.count('REAL64'), 1) + + def test_iso_use_multiple_distinct_specs(self): + """Two distinct specs → both appear in the same use line.""" + lines = _generate_ccpp_kinds({ + 'kind_phys': (_ISO, 'REAL64'), + 'kind_dyn': (_ISO, 'REAL32'), + }) + iso_line = next(l for l in lines if 'iso_fortran_env' in l) + self.assertIn('REAL32', iso_line) + self.assertIn('REAL64', iso_line) + + def test_host_module(self): + """Host-supplied module emits use of that module with renamed param.""" + lines = _generate_ccpp_kinds({ + 'kind_phys': ('my_host_kinds', 'kind_r8'), + }) + text = '\n'.join(lines) + self.assertIn('use my_host_kinds, only: kind_r8', text) + self.assertIn('kind_phys = kind_r8', text) + self.assertNotIn('iso_fortran_env', text) + + def test_mixed_modules(self): + """Mixed modules → one use line per module, sorted alphabetically.""" + lines = _generate_ccpp_kinds({ + 'kind_phys': ('my_host_kinds', 'kind_r8'), + 'kind_iso': (_ISO, 'REAL64'), + }) + use_lines = [l for l in lines if l.lstrip().startswith('use ')] + self.assertEqual(len(use_lines), 2) + # Sorted alphabetically: iso_fortran_env first, then my_host_kinds. + self.assertIn('iso_fortran_env', use_lines[0]) + self.assertIn('my_host_kinds', use_lines[1]) + + def test_multiple_kinds_sorted(self): + lines = _generate_ccpp_kinds({ + 'kind_z': (_ISO, 'REAL32'), + 'kind_a': (_ISO, 'REAL64'), + }) + param_lines = [l for l in lines if 'parameter' in l and 'public' in l] + self.assertEqual(len(param_lines), 2) + idx_a = next(i for i, l in enumerate(lines) if 'kind_a' in l and 'parameter' in l) + idx_z = next(i for i, l in enumerate(lines) if 'kind_z' in l and 'parameter' in l) + self.assertLess(idx_a, idx_z) + + def test_aligned_equals(self): + """Declarations should be column-aligned on '='.""" + lines = _generate_ccpp_kinds({ + 'kind_phys': (_ISO, 'REAL64'), + 'k': (_ISO, 'REAL32'), + }) + param_lines = [l for l in lines if 'parameter' in l and 'public' in l] + eq_positions = [l.index('=') for l in param_lines] + self.assertEqual(len(set(eq_positions)), 1, "All '=' should be at the same column") + + def test_implicit_none_private(self): + lines = _generate_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}) + self.assertIn(' implicit none', lines) + self.assertIn(' private', lines) + + def test_empty_raises(self): + with self.assertRaises(CCPPError): + _generate_ccpp_kinds({}) + + def test_header_comment(self): + lines = _generate_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}) + self.assertTrue(lines[0].startswith('!')) + self.assertIn('ccpp_kinds', lines[0]) + + def test_no_trailing_newlines_in_lines(self): + lines = _generate_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}) + for line in lines: + self.assertNotIn('\n', line) + + def test_integer_parameter(self): + lines = _generate_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}) + param_line = next(l for l in lines if 'kind_phys' in l and 'parameter' in l) + self.assertIn('integer', param_line) + self.assertIn('parameter', param_line) + self.assertIn('public', param_line) + + +class TestWriteCcppKinds(unittest.TestCase): + + def test_writes_file(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = write_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}, tmpdir) + self.assertTrue(os.path.isfile(path)) + self.assertEqual(os.path.basename(path), 'ccpp_kinds.F90') + + def test_file_content(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = write_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}, tmpdir) + with open(path) as fh: + content = fh.read() + self.assertIn('module ccpp_kinds', content) + self.assertIn('kind_phys', content) + self.assertIn('REAL64', content) + self.assertTrue(content.endswith('\n')) + + def test_creates_output_dir(self): + with tempfile.TemporaryDirectory() as tmpdir: + subdir = os.path.join(tmpdir, 'new_subdir') + write_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}, subdir) + self.assertTrue(os.path.isdir(subdir)) + + def test_returns_absolute_path(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = write_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}, tmpdir) + self.assertTrue(os.path.isabs(path)) + + +def load_tests(loader, tests, ignore): + import generator.kinds_writer as kw + tests.addTests(doctest.DocTestSuite(kw)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_legacy_compat.py b/unit-tests/test_legacy_compat.py new file mode 100644 index 00000000..1d154099 --- /dev/null +++ b/unit-tests/test_legacy_compat.py @@ -0,0 +1,296 @@ +"""Tests for the transient legacy-mode shim. + +This whole file is part of the legacy-compat migration shim and should +be deleted alongside ``metadata/legacy_compat.py`` when the migration +is complete. Search ``legacy-compat`` to find every touchpoint. +""" + +from __future__ import annotations + +import doctest +import io +import logging +import os +import sys +import unittest + +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_CAPGEN_DIR = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen') +if _CAPGEN_DIR not in sys.path: + sys.path.insert(0, _CAPGEN_DIR) + +from metadata import legacy_compat # noqa: E402 +from metadata.metadata_table import ( # noqa: E402 + MetaVar, _parse_dimensions, _parse_lines, +) +from metadata.parse_tools import CCPPError, ParseContext # noqa: E402 + + +def _ctx(): + return ParseContext(linenum=1, filename='legacy_compat_test.meta') + + +class _LegacyModeFixture(unittest.TestCase): + """Mixin that flips legacy mode on for the duration of a test and + guarantees it goes back off afterwards (the flag is process state). + """ + + def setUp(self): + # Sanity: never start a test with the flag set by an earlier + # test that crashed before cleanup. + legacy_compat.disable() + + def tearDown(self): + legacy_compat.disable() + + +class TestTranslateOff(_LegacyModeFixture): + """When legacy mode is disabled, ``translate`` is a strict identity.""" + + def test_identity_on_legacy_name_when_disabled(self): + self.assertFalse(legacy_compat.is_enabled()) + self.assertEqual( + legacy_compat.translate('horizontal_loop_extent'), + 'horizontal_loop_extent', + ) + + def test_identity_on_unknown_name(self): + self.assertEqual( + legacy_compat.translate('air_temperature'), 'air_temperature', + ) + + +class TestEnableDisable(_LegacyModeFixture): + + def test_enable_flips_flag_and_writes_banner(self): + sink = io.StringIO() + legacy_compat.enable(_stream=sink) + self.assertTrue(legacy_compat.is_enabled()) + out = sink.getvalue() + # Bold banner: starred border, the deprecated names, and the + # canonical replacements all appear. + self.assertIn('LEGACY-MODE ENABLED', out) + self.assertIn('horizontal_loop_extent', out) + self.assertIn('horizontal_dimension', out) + # Banner also enumerates the number_of_openmp_threads pair. + self.assertIn('number_of_openmp_threads', out) + self.assertIn('number_of_threads', out) + self.assertIn('TRANSIENT', out) + self.assertIn('REMOVED', out) + self.assertGreaterEqual(out.count('*' * 10), 2) + + def test_enable_is_idempotent(self): + sink1 = io.StringIO() + legacy_compat.enable(_stream=sink1) + first = sink1.getvalue() + sink2 = io.StringIO() + legacy_compat.enable(_stream=sink2) # second call — no banner + self.assertEqual(sink2.getvalue(), '') + self.assertIn('LEGACY-MODE', first) + + def test_disable_resets(self): + legacy_compat.enable(_stream=io.StringIO()) + self.assertTrue(legacy_compat.is_enabled()) + legacy_compat.disable() + self.assertFalse(legacy_compat.is_enabled()) + + def test_logger_receives_warning(self): + logger = logging.getLogger('legacy_compat_test_logger') + records = [] + + class _Capture(logging.Handler): + def emit(self, record): + records.append(record) + + handler = _Capture(level=logging.WARNING) + logger.addHandler(handler) + try: + legacy_compat.enable(logger=logger, _stream=io.StringIO()) + self.assertEqual(len(records), 1) + self.assertEqual(records[0].levelno, logging.WARNING) + msg = records[0].getMessage() + self.assertIn('horizontal_loop_extent', msg) + self.assertIn('number_of_openmp_threads', msg) + finally: + logger.removeHandler(handler) + + +class TestTranslateOn(_LegacyModeFixture): + """When legacy mode is enabled, the documented map applies.""" + + def setUp(self): + super().setUp() + legacy_compat.enable(_stream=io.StringIO()) + + def test_horizontal_loop_extent_rewritten(self): + self.assertEqual( + legacy_compat.translate('horizontal_loop_extent'), + 'horizontal_dimension', + ) + + def test_number_of_openmp_threads_rewritten(self): + """Legacy CCPP-physics hosts (and SCM 17p8) size per-thread DDT + containers by ``number_of_openmp_threads``. The capgen + convention is ``number_of_threads`` (matching the + ``thread_number`` control variable name). Legacy mode rewrites + both as a standard_name attribute AND as a dimension token.""" + self.assertEqual( + legacy_compat.translate('number_of_openmp_threads'), + 'number_of_threads', + ) + + def test_unknown_name_passes_through(self): + self.assertEqual( + legacy_compat.translate('air_temperature'), 'air_temperature', + ) + + def test_uppercase_legacy_passes_through_translate(self): + # ``translate`` itself is case-sensitive — case folding is the + # caller's responsibility (it happens upstream in + # ``check_cf_standard_name``). This pins that contract so a + # future refactor doesn't accidentally widen it. + self.assertEqual( + legacy_compat.translate('Horizontal_Loop_Extent'), + 'Horizontal_Loop_Extent', + ) + + +######################################################################## +# Integration through metadata_table hook points +######################################################################## + +class TestMetaVarStandardNameHook(_LegacyModeFixture): + """``MetaVar.set_attr('standard_name', ...)`` runs the value through + ``check_cf_standard_name`` (which lowercases) AND through + ``legacy_compat.translate``. When legacy mode is enabled, a scheme + declaring ``standard_name = horizontal_loop_extent`` ends up with + ``standard_name == 'horizontal_dimension'``.""" + + def _make_var(self, std_name_value): + ctx = _ctx() + v = MetaVar('foo', ctx) + v.set_attr('standard_name', std_name_value, ctx) + return v + + def test_disabled_keeps_legacy_name(self): + # Default mode: the parser accepts the legacy name (it's a + # valid CF identifier) but does NOT rewrite it. Downstream + # consumers reject it just like they would in a non-legacy + # build. + v = self._make_var('horizontal_loop_extent') + self.assertEqual(v.standard_name, 'horizontal_loop_extent') + + def test_enabled_rewrites_lowercase_legacy_name(self): + legacy_compat.enable(_stream=io.StringIO()) + v = self._make_var('horizontal_loop_extent') + self.assertEqual(v.standard_name, 'horizontal_dimension') + + def test_enabled_rewrites_mixedcase_legacy_name(self): + # check_cf_standard_name lowercases first; translate runs + # against the lowercase form. Mixed-case legacy spellings + # therefore still get rewritten. + legacy_compat.enable(_stream=io.StringIO()) + v = self._make_var('Horizontal_Loop_Extent') + self.assertEqual(v.standard_name, 'horizontal_dimension') + + def test_enabled_leaves_non_legacy_unchanged(self): + legacy_compat.enable(_stream=io.StringIO()) + v = self._make_var('air_temperature') + self.assertEqual(v.standard_name, 'air_temperature') + + +class TestDimensionHook(_LegacyModeFixture): + """``_parse_dimensions`` runs each non-integer token through + ``legacy_compat.translate`` after lowercasing.""" + + def test_disabled_keeps_legacy_dim_token(self): + dims = _parse_dimensions( + '(horizontal_loop_extent, vertical_layer_dimension)', _ctx(), + ) + self.assertEqual(dims, + ['horizontal_loop_extent', 'vertical_layer_dimension']) + + def test_enabled_rewrites_legacy_dim_token(self): + legacy_compat.enable(_stream=io.StringIO()) + dims = _parse_dimensions( + '(horizontal_loop_extent, vertical_layer_dimension)', _ctx(), + ) + self.assertEqual(dims, + ['horizontal_dimension', 'vertical_layer_dimension']) + + def test_enabled_rewrites_inside_range_form(self): + legacy_compat.enable(_stream=io.StringIO()) + dims = _parse_dimensions( + '(ccpp_constant_one:horizontal_loop_extent)', _ctx(), + ) + self.assertEqual(dims, ['ccpp_constant_one:horizontal_dimension']) + + def test_enabled_passes_integer_literals_through(self): + legacy_compat.enable(_stream=io.StringIO()) + dims = _parse_dimensions('(1:8)', _ctx()) + self.assertEqual(dims, ['1:8']) + + def test_enabled_mixed_case_legacy_in_dim(self): + legacy_compat.enable(_stream=io.StringIO()) + dims = _parse_dimensions('(Horizontal_Loop_Extent)', _ctx()) + self.assertEqual(dims, ['horizontal_dimension']) + + +class TestEndToEndMetadataParse(_LegacyModeFixture): + """Full parse of a small scheme metadata snippet via ``_parse_lines`` + confirms that the legacy name flows through both hook points + (standard_name on a scalar arg + dim token on an array arg).""" + + _META = ( + '[ccpp-table-properties]\n' + ' name = legacy_test_scheme\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = legacy_test_scheme_run\n' + ' type = scheme\n' + '[ ncols ]\n' + ' standard_name = horizontal_loop_extent\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + ' intent = in\n' + '[ t ]\n' + ' standard_name = air_temperature\n' + ' units = K\n' + ' dimensions = (horizontal_loop_extent, vertical_layer_dimension)\n' + ' type = real | kind = kind_phys\n' + ' intent = inout\n' + ) + + def _parse_and_get_section(self): + tables = _parse_lines(self._META.splitlines(keepends=True), 't.meta') + return tables[0].sections()[0] + + def test_legacy_off_preserves_legacy_name(self): + section = self._parse_and_get_section() + std_names = [v.standard_name for v in section.variables] + self.assertIn('horizontal_loop_extent', std_names) + dim_tokens = section.variables[1].dimensions + self.assertEqual(dim_tokens[0], 'horizontal_loop_extent') + + def test_legacy_on_rewrites_both_sites(self): + legacy_compat.enable(_stream=io.StringIO()) + section = self._parse_and_get_section() + std_names = [v.standard_name for v in section.variables] + self.assertIn('horizontal_dimension', std_names) + self.assertNotIn('horizontal_loop_extent', std_names) + dim_tokens = section.variables[1].dimensions + self.assertEqual(dim_tokens[0], 'horizontal_dimension') + + +######################################################################## +# Doctest loader for legacy_compat module +######################################################################## + +def load_tests(loader, tests, ignore): + tests.addTests(doctest.DocTestSuite(legacy_compat)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_metadata_table.py b/unit-tests/test_metadata_table.py new file mode 100644 index 00000000..ea69c3d1 --- /dev/null +++ b/unit-tests/test_metadata_table.py @@ -0,0 +1,2231 @@ +#!/usr/bin/env python3 + +"""Unit tests for :mod:`metadata.metadata_table`. + +Run with:: + + python -m pytest capgen/tests/test_metadata_table.py -v + +or:: + + python -m unittest capgen.tests.test_metadata_table + +All test methods follow the ``test_`` naming convention and are +documented inline to explain both what is being tested and *why* it matters +for the redesigned generator. +""" + +import os +import sys +import textwrap +import tempfile +import unittest + +# ---- locate the package root ----------------------------------------------- +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_PKG_ROOT = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen') +if _PKG_ROOT not in sys.path: + sys.path.insert(0, _PKG_ROOT) + +# ---- imports from the package ----------------------------------------------- +from metadata.parse_tools import CCPPError, ParseSyntaxError +from metadata.metadata_table import ( + MetaVar, + MetadataSection, + MetadataTable, + parse_metadata_file, + _parse_lines, + VALID_TABLE_TYPES, + VALID_SCHEME_PHASES, + VALID_INTENTS, + SCHEME_TABLE_TYPE, + SINGLETON_TABLE_TYPES, + _is_blank, + _parse_bool, + _parse_dimensions, + _check_var_type, + _parse_config_line, + _strip_inline_comment, +) +from metadata.parse_tools.parse_source import ParseContext + +# ---- sample file directory -------------------------------------------------- +_SAMPLE_DIR = os.path.join(_TESTS_DIR, 'sample_files') + + +######################################################################## +# Helper utilities +######################################################################## + +def _ctx(lineno=0): + """Return a minimal :class:`ParseContext` for testing.""" + return ParseContext(linenum=lineno, filename='test_file.meta') + + +def _parse_text(text: str): + """Parse a metadata string and return the list of MetadataTable objects.""" + lines = textwrap.dedent(text).splitlines(keepends=True) + return _parse_lines(lines, '') + + +######################################################################## +# Helper function tests +######################################################################## + +class TestIsBlank(unittest.TestCase): + """Tests for :func:`_is_blank`.""" + + def test_empty_string(self): + self.assertTrue(_is_blank('')) + + def test_whitespace_only(self): + self.assertTrue(_is_blank(' \t ')) + + def test_hash_comment(self): + self.assertTrue(_is_blank('# this is a comment')) + + def test_semicolon_comment(self): + self.assertTrue(_is_blank('; comment')) + + def test_hash_with_leading_spaces(self): + self.assertTrue(_is_blank(' # indented comment')) + + def test_real_content(self): + self.assertFalse(_is_blank('name = foo')) + + def test_bracket_header(self): + self.assertFalse(_is_blank('[ccpp-table-properties]')) + + +class TestStripInlineComment(unittest.TestCase): + """Tests for :func:`_strip_inline_comment` — trailing ``# ...`` is + a comment that the parser must discard before any other handling. + """ + + def test_plain_line_unchanged(self): + self.assertEqual( + _strip_inline_comment('dimensions = (horizontal_dimension)'), + 'dimensions = (horizontal_dimension)', + ) + + def test_strips_trailing_hash_comment(self): + self.assertEqual( + _strip_inline_comment('dimensions = () # (nap_indices)'), + 'dimensions = ()', + ) + + def test_strips_section_header_comment(self): + self.assertEqual( + _strip_inline_comment('[ ap_indices ] # legacy'), + '[ ap_indices ]', + ) + + def test_full_line_comment_collapses_to_empty(self): + self.assertEqual(_strip_inline_comment('# whole line'), '') + + def test_hash_at_column_zero(self): + self.assertEqual(_strip_inline_comment('#x'), '') + + +class TestInlineCommentInParser(unittest.TestCase): + """End-to-end check that the parser ignores trailing ``#`` comments + on any metadata line — the user-reported bug surfaced on a + ``dimensions =`` attribute value but the fix is universal.""" + + _SRC = ( + '[ccpp-table-properties]\n' + ' name = mod # the host module\n' + ' type = host\n' + '[ccpp-arg-table]\n' + ' name = mod\n' + ' type = host\n' + '[ ap_indices ] # legacy index slot\n' + ' standard_name = ap_indices\n' + ' units = index\n' + ' dimensions = () # (nap_indices)\n' + ' type = integer\n' + ) + + def test_parses_cleanly(self): + tables = _parse_text(self._SRC) + self.assertEqual(len(tables), 1) + sec = tables[0].sections()[0] + var = sec.variables[0] + self.assertEqual(var.local_name, 'ap_indices') + self.assertEqual(var.dimensions, []) + self.assertEqual(var.type, 'integer') + self.assertEqual(tables[0].table_name, 'mod') + + +class TestParseBool(unittest.TestCase): + """Tests for :func:`_parse_bool`.""" + + def test_true_variants(self): + ctx = _ctx() + for val in ('True', 'true', 'TRUE', '.true.', '.TRUE.', 't', '1'): + with self.subTest(val=val): + self.assertTrue(_parse_bool(val, ctx)) + + def test_false_variants(self): + ctx = _ctx() + for val in ('False', 'false', 'FALSE', '.false.', '.FALSE.', 'f', '0'): + with self.subTest(val=val): + self.assertFalse(_parse_bool(val, ctx)) + + def test_invalid(self): + ctx = _ctx() + with self.assertRaises(CCPPError): + _parse_bool('yes', ctx) + + +class TestParseDimensions(unittest.TestCase): + """Tests for :func:`_parse_dimensions`.""" + + def test_scalar(self): + self.assertEqual(_parse_dimensions('()', _ctx()), []) + + def test_one_dim(self): + self.assertEqual( + _parse_dimensions('(horizontal_dimension)', _ctx()), + ['horizontal_dimension'] + ) + + def test_two_dims(self): + result = _parse_dimensions( + '(horizontal_dimension, vertical_layer_dimension)', _ctx() + ) + self.assertEqual(result, ['horizontal_dimension', 'vertical_layer_dimension']) + + def test_whitespace_inside(self): + result = _parse_dimensions(' ( dim1 , dim2 ) ', _ctx()) + self.assertEqual(result, ['dim1', 'dim2']) + + def test_missing_parens(self): + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_dimensions('dim1, dim2', _ctx()) + + def test_empty_entry(self): + """An empty entry like ``(,dim2)`` must be rejected.""" + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_dimensions('(, dim2)', _ctx()) + + def test_mixed_case_normalised_to_lower(self): + """CCPP standard names are case-insensitive; the parser must + normalise dimension tokens to lower-case so downstream + host_dict lookups (which store std names lower-cased per + :func:`check_cf_standard_name`) succeed.""" + result = _parse_dimensions( + '(number_of_aerosol_tracers_MG)', _ctx(), + ) + self.assertEqual(result, ['number_of_aerosol_tracers_mg']) + + def test_mixed_case_in_range_normalised(self): + """Range form ``lower:upper`` lowercases both halves.""" + result = _parse_dimensions( + '(ccpp_constant_one:Vertical_LAYER_dimension)', _ctx(), + ) + self.assertEqual( + result, ['ccpp_constant_one:vertical_layer_dimension'], + ) + + def test_integer_literal_passes_through(self): + """Integer-literal dim entries (used in DDT field shapes) are + unaffected by the lower-case normalisation.""" + result = _parse_dimensions('(8)', _ctx()) + self.assertEqual(result, ['8']) + + +class TestCheckVarType(unittest.TestCase): + """Tests for :func:`_check_var_type`.""" + + def test_intrinsic_types(self): + for t in ('real', 'integer', 'logical', 'character', 'complex'): + with self.subTest(t=t): + self.assertEqual(_check_var_type(t, _ctx()), t) + + def test_ddt_identifier(self): + self.assertEqual(_check_var_type('gfs_statein_type', _ctx()), + 'gfs_statein_type') + + def test_type_parens_form(self): + result = _check_var_type('type(my_ddt)', _ctx()) + self.assertEqual(result, 'type(my_ddt)') + + def test_external_type(self): + result = _check_var_type('external:mpi_f08:mpi_comm', _ctx()) + self.assertEqual(result, 'external:mpi_f08:mpi_comm') + + def test_invalid_type(self): + with self.assertRaises((CCPPError, ParseSyntaxError)): + _check_var_type('123invalid', _ctx()) + + +class TestParseConfigLine(unittest.TestCase): + """Tests for :func:`_parse_config_line`.""" + + def test_simple_pair(self): + result = _parse_config_line(' name = foo ', _ctx()) + self.assertEqual(result, [('name', 'foo')]) + + def test_pipe_separator(self): + result = _parse_config_line('units = 1 | dimensions = ()', _ctx()) + self.assertEqual(result, [('units', '1'), ('dimensions', '()')]) + + def test_blank_line(self): + self.assertEqual(_parse_config_line('', _ctx()), []) + + def test_comment_line(self): + self.assertEqual(_parse_config_line('# comment', _ctx()), []) + + def test_bad_line(self): + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_config_line('no_equals_sign', _ctx()) + + +######################################################################## +# MetaVar tests +######################################################################## + +class TestMetaVar(unittest.TestCase): + """Tests for :class:`MetaVar`.""" + + def _make_var(self, local_name='my_var', **attrs): + """Build a MetaVar with sensible defaults plus any overrides.""" + ctx = _ctx() + var = MetaVar(local_name, ctx) + defaults = { + 'standard_name': 'my_standard_name', + 'units': '1', + 'dimensions': '()', + 'type': 'integer', + } + defaults.update(attrs) + for k, v in defaults.items(): + var.set_attr(k, v, ctx) + return var + + def test_creation(self): + var = self._make_var() + self.assertEqual(var.local_name, 'my_var') + self.assertEqual(var.standard_name, 'my_standard_name') + + def test_standard_name_lowercased(self): + """Standard names must be CF names (lowercased by the checker).""" + ctx = _ctx() + var = MetaVar('v', ctx) + var.set_attr('standard_name', 'Horizontal_Dimension', ctx) + self.assertEqual(var.standard_name, 'horizontal_dimension') + + def test_dimensions_scalar(self): + var = self._make_var(dimensions='()') + self.assertEqual(var.dimensions, []) + + def test_dimensions_array(self): + var = self._make_var(dimensions='(horizontal_dimension, vertical_layer_dimension)') + self.assertEqual(var.dimensions, + ['horizontal_dimension', 'vertical_layer_dimension']) + + def test_intent_valid(self): + ctx = _ctx() + for intent in ('in', 'out', 'inout'): + with self.subTest(intent=intent): + var = self._make_var() + var.set_attr('intent', intent, ctx) + self.assertEqual(var.intent, intent) + + def test_intent_invalid(self): + ctx = _ctx() + var = self._make_var() + with self.assertRaises(CCPPError): + var.set_attr('intent', 'banana', ctx) + + def test_invalid_attribute_error_carries_full_context(self): + """When a check_X helper rejects a value, the resulting CCPPError + MUST name the offending variable, the attribute, the raw value, + AND the source location. Without this enrichment the user sees + a bare ``'' is not a valid unit`` with no clue which file/line/var + is at fault — every check_X helper is unaware of context. + Regression for the SCM ccpp-physics 61-file parse where the user + couldn't locate the offending metadata. + """ + ctx = ParseContext(linenum=42, filename='broken_scheme.meta') + var = MetaVar('my_bad_var', ctx) + with self.assertRaises(CCPPError) as raised: + var.set_attr('units', '', ctx) + msg = str(raised.exception) + self.assertIn("'my_bad_var'", msg) # variable name + self.assertIn("'units'", msg) # attribute name + self.assertIn("broken_scheme.meta", msg) # file + self.assertIn(":43", msg) # line (1-based) + self.assertIn("not a valid unit", msg) # inner reason + + def test_invalid_attribute_error_does_not_double_wrap(self): + """Helpers that already include the location (``_parse_dimensions``, + ``_check_var_type``) should not have their location duplicated in + the wrapper message. Test confirms the wrapper detects the + already-present location and re-raises unchanged.""" + ctx = ParseContext(linenum=5, filename='dim_broken.meta') + var = MetaVar('v', ctx) + with self.assertRaises(CCPPError) as raised: + var.set_attr('dimensions', '(this is malformed', ctx) + msg = str(raised.exception) + # Location appears exactly once (no nested duplication). + self.assertEqual(msg.count('dim_broken.meta'), 1) + + def test_protected_bool(self): + var = self._make_var(protected='True') + self.assertTrue(var.protected) + + def test_optional_bool(self): + var = self._make_var(optional='False') + self.assertFalse(var.optional) + + def test_local_name_accepts_long_subscript_index(self): + """Sliced local names like ``dqdt(:,:,index_of_)`` + carry CCPP standard names as subscript tokens; those routinely + exceed the 63-char Fortran identifier limit. Only the base + identifier (``dqdt``) needs to fit the limit — the subscript is + a standard-name reference resolved separately. + """ + long_std = ('index_of_cloud_liquid_water_mixing_ratio_' + 'in_tracer_concentration_array') # 67 chars + self.assertGreater(len(long_std), 63) + long_local = 'dqdt(:,:,{})'.format(long_std) + # Must NOT raise. + var = self._make_var(local_name=long_local) + self.assertEqual(var.local_name, long_local) + + def test_local_name_base_still_length_checked(self): + """The base identifier (everything before the first ``(``) must + still fit the 63-char limit — only subscript tokens are exempt. + """ + from metadata.parse_tools import ParseSyntaxError + long_base = 'a' * 64 # 64 > FORTRAN_MAX_IDENT_LEN + with self.assertRaises(ParseSyntaxError): + MetaVar(long_base, _ctx()) + # Same long base inside a slice — still rejected. + with self.assertRaises(ParseSyntaxError): + MetaVar('{}(:)'.format(long_base), _ctx()) + + def test_local_name_rejects_malformed_reference(self): + """The form check still applies — only the per-token length + limit was relaxed. A malformed reference is still an error.""" + from metadata.parse_tools import ParseSyntaxError + with self.assertRaises(ParseSyntaxError): + MetaVar('not a valid id', _ctx()) + + def test_top_at_one_default_false(self): + var = self._make_var() + self.assertFalse(var.top_at_one) + + def test_top_at_one_true(self): + for value in ('True', '.true.', 'true'): + with self.subTest(value=value): + var = self._make_var(top_at_one=value) + self.assertTrue(var.top_at_one) + + def test_top_at_one_false(self): + for value in ('False', '.false.', 'false'): + with self.subTest(value=value): + var = self._make_var(top_at_one=value) + self.assertFalse(var.top_at_one) + + def test_diagnostic_name_explicit(self): + var = self._make_var(diagnostic_name='temperature') + self.assertEqual(var.diagnostic_name, 'temperature') + self.assertEqual(var.diagnostic_name_fixed, '') + + def test_diagnostic_name_template_accepted(self): + var = self._make_var(diagnostic_name='foo_${scheme_name}') + self.assertEqual(var.diagnostic_name, 'foo_${scheme_name}') + + def test_diagnostic_name_fixed_explicit(self): + var = self._make_var(diagnostic_name_fixed='Q') + self.assertEqual(var.diagnostic_name_fixed, 'Q') + # When _fixed is set, diagnostic_name returns '' (no local_name fallback). + self.assertEqual(var.diagnostic_name, '') + + def test_diagnostic_name_defaults_to_local_name(self): + """Neither attribute set: diagnostic_name defaults to local_name.""" + var = self._make_var(local_name='my_local_var') + self.assertEqual(var.diagnostic_name, 'my_local_var') + self.assertEqual(var.diagnostic_name_fixed, '') + + def test_diagnostic_name_invalid(self): + ctx = _ctx() + var = self._make_var() + with self.assertRaises(CCPPError): + var.set_attr('diagnostic_name', 'pref_${scheme}_suff', ctx) + + def test_diagnostic_name_fixed_invalid(self): + ctx = _ctx() + var = self._make_var() + with self.assertRaises(CCPPError): + var.set_attr('diagnostic_name_fixed', '2bad', ctx) + + def test_diagnostic_name_mutually_exclusive(self): + """Setting both diagnostic_name and diagnostic_name_fixed is an error.""" + ctx = _ctx() + var = self._make_var(diagnostic_name='temperature') + with self.assertRaises(CCPPError): + var.set_attr('diagnostic_name_fixed', 'Q', ctx) + + def test_diagnostic_name_fixed_then_name_mutually_exclusive(self): + """Order independence: fixed first, then name, also rejected.""" + ctx = _ctx() + var = self._make_var(diagnostic_name_fixed='Q') + with self.assertRaises(CCPPError): + var.set_attr('diagnostic_name', 'temperature', ctx) + + def test_duplicate_attr(self): + """Setting the same attribute twice is an error.""" + ctx = _ctx() + var = MetaVar('v', ctx) + var.set_attr('standard_name', 'foo', ctx) + with self.assertRaises(CCPPError): + var.set_attr('standard_name', 'bar', ctx) + + def test_unknown_attr(self): + ctx = _ctx() + var = MetaVar('v', ctx) + with self.assertRaises(CCPPError): + var.set_attr('banana', 'value', ctx) + + def test_validate_requires_intent_for_scheme(self): + """Scheme variables need intent; missing it must raise CCPPError.""" + ctx = _ctx() + var = self._make_var() # no intent set + with self.assertRaises(CCPPError): + var.validate(require_intent=True, context=ctx) + + def test_validate_no_intent_for_host(self): + """Host variables do not need intent; validate must pass without it.""" + ctx = _ctx() + var = self._make_var() + var.validate(require_intent=False, context=ctx) # should not raise + + def test_validate_missing_units_defaults_to_none(self): + """Omitting ``units`` is allowed; it defaults to ``'none'``.""" + ctx = _ctx() + var = MetaVar('v', ctx) + var.set_attr('standard_name', 'foo', ctx) + var.set_attr('dimensions', '()', ctx) + var.set_attr('type', 'integer', ctx) + var.validate(require_intent=False, context=ctx) # should not raise + self.assertEqual(var.units, 'none') + + def test_external_ddt_helpers(self): + var = self._make_var(type='external:mpi_f08:mpi_comm') + self.assertTrue(var.is_external_ddt()) + self.assertEqual(var.external_ddt_module(), 'mpi_f08') + self.assertEqual(var.external_ddt_typename(), 'mpi_comm') + + def test_non_external_ddt_helpers(self): + var = self._make_var(type='real') + self.assertFalse(var.is_external_ddt()) + self.assertIsNone(var.external_ddt_module()) + self.assertIsNone(var.external_ddt_typename()) + + def test_invalid_local_name(self): + """Local name must be a valid Fortran identifier.""" + with self.assertRaises((CCPPError, ParseSyntaxError)): + MetaVar('123bad', _ctx()) + + def test_local_name_too_long(self): + """Local name must not exceed 63 characters (Fortran limit).""" + long_name = 'a' * 64 + with self.assertRaises((CCPPError, ParseSyntaxError)): + MetaVar(long_name, _ctx()) + + +######################################################################## +# MetadataSection tests +######################################################################## + +class TestMetadataSection(unittest.TestCase): + """Tests for :class:`MetadataSection`.""" + + def _make_scheme_section(self, phase='run', scheme_name='my_scheme'): + ctx = _ctx() + return MetadataSection( + section_name='{}__{}'.format(scheme_name, phase).replace('__', '_'), + section_type='scheme', + table_name=scheme_name, + context=ctx, + ) + + def test_scheme_phase_extraction(self): + for phase in VALID_SCHEME_PHASES: + with self.subTest(phase=phase): + sec = self._make_scheme_section(phase=phase) + self.assertEqual(sec.phase, phase) + + def test_finalize_rejected(self): + """'finalize' was renamed to 'final'; the old name must be rejected.""" + ctx = _ctx() + with self.assertRaises(CCPPError) as cm: + MetadataSection('my_scheme_finalize', 'scheme', 'my_scheme', ctx) + self.assertIn('final', str(cm.exception)) + + def test_host_section_has_no_phase(self): + ctx = _ctx() + sec = MetadataSection('physics_data', 'host', 'physics_data', ctx) + self.assertIsNone(sec.phase) + + def test_duplicate_standard_name(self): + """Adding two variables with the same standard name must raise.""" + ctx = _ctx() + sec = MetadataSection('my_scheme_run', 'scheme', 'my_scheme', ctx) + var1 = MetaVar('a_var', ctx) + for attr, val in [('standard_name', 'foo'), ('units', '1'), + ('dimensions', '()'), ('type', 'integer'), + ('intent', 'in')]: + var1.set_attr(attr, val, ctx) + var1.validate(require_intent=True, context=ctx) + sec.add_variable(var1) + + var2 = MetaVar('b_var', ctx) + for attr, val in [('standard_name', 'foo'), ('units', '1'), + ('dimensions', '()'), ('type', 'integer'), + ('intent', 'in')]: + var2.set_attr(attr, val, ctx) + var2.validate(require_intent=True, context=ctx) + with self.assertRaises(CCPPError): + sec.add_variable(var2) + + def test_invalid_table_type(self): + ctx = _ctx() + with self.assertRaises(CCPPError): + MetadataSection('t', 'banana', 't', ctx) + + def test_scheme_name_mismatch(self): + """Section name not starting with scheme name must raise.""" + ctx = _ctx() + with self.assertRaises(CCPPError): + MetadataSection('other_scheme_run', 'scheme', 'my_scheme', ctx) + + +######################################################################## +# MetadataTable tests +######################################################################## + +class TestMetadataTable(unittest.TestCase): + """Tests for :class:`MetadataTable`.""" + + def test_valid_types(self): + for ttype in VALID_TABLE_TYPES: + with self.subTest(ttype=ttype): + ctx = _ctx() + tbl = MetadataTable('tbl', ttype, 'f.meta', ctx) + self.assertEqual(tbl.table_type, ttype) + + def test_invalid_type(self): + ctx = _ctx() + with self.assertRaises(CCPPError): + MetadataTable('t', 'module', 'f.meta', ctx) + + def test_is_scheme(self): + ctx = _ctx() + self.assertTrue(MetadataTable('s', 'scheme', 'f.meta', ctx).is_scheme) + self.assertFalse(MetadataTable('h', 'host', 'f.meta', ctx).is_scheme) + + def test_singleton_allows_one_section(self): + """Singleton table types (host, control, ddt, suite) allow only one section.""" + for ttype in SINGLETON_TABLE_TYPES: + with self.subTest(ttype=ttype): + ctx = _ctx() + tbl = MetadataTable('t', ttype, 'f.meta', ctx) + sec1 = MetadataSection('t', ttype, 't', ctx) + tbl.add_section(sec1) + sec2 = MetadataSection('t', ttype, 't', ctx) + with self.assertRaises(CCPPError): + tbl.add_section(sec2) + + def test_scheme_allows_multiple_sections(self): + """Scheme tables allow one section per phase.""" + ctx = _ctx() + tbl = MetadataTable('s', 'scheme', 'f.meta', ctx) + for phase in ('init', 'run', 'final'): + sec = MetadataSection('s_{}'.format(phase), 'scheme', 's', ctx) + tbl.add_section(sec) + self.assertEqual(len(tbl.sections()), 3) + + def test_section_type_mismatch(self): + """Section type must match table type.""" + ctx = _ctx() + tbl = MetadataTable('my_host', 'host', 'f.meta', ctx) + # The section name must be valid for scheme (scheme_name_phase); + # use a different scheme name so MetadataSection construction succeeds, + # and the CCPPError is raised only at add_section() due to type mismatch. + sec = MetadataSection('some_scheme_run', 'scheme', 'some_scheme', ctx) + with self.assertRaises(CCPPError): + tbl.add_section(sec) + + def test_section_for_phase(self): + ctx = _ctx() + tbl = MetadataTable('s', 'scheme', 'f.meta', ctx) + run_sec = MetadataSection('s_run', 'scheme', 's', ctx) + tbl.add_section(run_sec) + self.assertIs(tbl.section_for_phase('run'), run_sec) + self.assertIsNone(tbl.section_for_phase('init')) + + +######################################################################## +# parse_metadata_file / _parse_lines tests +######################################################################## + +class TestParseLines(unittest.TestCase): + """Tests for the actual ini-file parser via :func:`_parse_lines`.""" + + # ---- valid cases ------------------------------------------------------- + + def test_host_table(self): + text = """ + [ccpp-table-properties] + name = my_host + type = host + + [ccpp-arg-table] + name = my_host + type = host + [ im ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + """ + tables = _parse_text(text) + self.assertEqual(len(tables), 1) + tbl = tables[0] + self.assertEqual(tbl.table_name, 'my_host') + self.assertEqual(tbl.table_type, 'host') + self.assertEqual(len(tbl.sections()), 1) + sec = tbl.sections()[0] + self.assertEqual(len(sec.variables), 1) + var = sec.variables[0] + self.assertEqual(var.local_name, 'im') + self.assertEqual(var.standard_name, 'horizontal_dimension') + self.assertEqual(var.dimensions, []) + self.assertIsNone(var.intent) + + def test_scheme_three_phases(self): + text = """ + [ccpp-table-properties] + name = my_scheme + type = scheme + + [ccpp-arg-table] + name = my_scheme_init + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ im ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + intent = in + + [ccpp-arg-table] + name = my_scheme_final + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """ + tables = _parse_text(text) + self.assertEqual(len(tables), 1) + tbl = tables[0] + self.assertTrue(tbl.is_scheme) + self.assertEqual(len(tbl.sections()), 3) + phases = {sec.phase for sec in tbl.sections()} + self.assertEqual(phases, {'init', 'run', 'final'}) + + def test_two_tables_in_one_file(self): + """A .meta file may contain one DDT table followed by a scheme table.""" + text = """ + [ccpp-table-properties] + name = my_ddt + type = ddt + + [ccpp-arg-table] + name = my_ddt + type = ddt + [ field1 ] + standard_name = some_field + units = 1 + dimensions = () + type = real + + [ccpp-table-properties] + name = my_scheme + type = scheme + + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ im ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + intent = in + """ + tables = _parse_text(text) + self.assertEqual(len(tables), 2) + self.assertEqual(tables[0].table_type, 'ddt') + self.assertEqual(tables[1].table_type, 'scheme') + + def test_pipe_separated_attributes(self): + """Multiple attributes on one line with ``|`` separator.""" + text = """ + [ccpp-table-properties] + name = s + type = scheme + + [ccpp-arg-table] + name = s_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none | dimensions = () | type = character | kind = len=512 + intent = out + """ + tables = _parse_text(text) + var = tables[0].sections()[0].variables[0] + self.assertEqual(var.units, 'none') + self.assertEqual(var.dimensions, []) + self.assertEqual(var.type, 'character') + self.assertEqual(var.kind, 'len=512') + self.assertEqual(var.intent, 'out') + + def test_control_table(self): + text = """ + [ccpp-table-properties] + name = ctrl + type = control + + [ccpp-arg-table] + name = ctrl + type = control + [ tnum ] + standard_name = thread_number + units = 1 + dimensions = () + type = integer + """ + tables = _parse_text(text) + self.assertEqual(tables[0].table_type, 'control') + + def test_ddt_instance_in_host_table(self): + """Host table declaring a DDT instance variable (array of DDT).""" + text = """ + [ccpp-table-properties] + name = CCPP_data + type = host + + [ccpp-arg-table] + name = CCPP_data + type = host + [ gfs_statein ] + standard_name = gfs_statein + units = none + dimensions = (number_of_instances) + type = gfs_statein_type + """ + tables = _parse_text(text) + var = tables[0].sections()[0].variables[0] + self.assertEqual(var.type, 'gfs_statein_type') + self.assertEqual(var.dimensions, ['number_of_instances']) + + def test_comments_and_blank_lines_ignored(self): + """Comment lines (``#``) and blank lines must be skipped silently.""" + text = """ + # This is a comment + + [ccpp-table-properties] + name = h + type = host + + ######################################## + [ccpp-arg-table] + # section comment + name = h + type = host + ; semicolon comment + [ x ] + standard_name = some_var + units = 1 + dimensions = () + type = integer + """ + tables = _parse_text(text) + self.assertEqual(len(tables), 1) + self.assertEqual(len(tables[0].sections()[0].variables), 1) + + # ---- error cases ------------------------------------------------------- + + def test_module_type_rejected(self): + """``type = module`` must produce a CCPPError mentioning 'host'.""" + text = """ + [ccpp-table-properties] + name = physics_mod + type = module + """ + with self.assertRaises(CCPPError) as cm: + _parse_text(text) + self.assertIn('host', str(cm.exception).lower()) + + def test_banana_type_rejected(self): + """An unknown table type must raise CCPPError.""" + text = """ + [ccpp-table-properties] + name = bad + type = banana + """ + with self.assertRaises(CCPPError): + _parse_text(text) + + def test_finalize_phase_rejected(self): + """``_finalize`` phase name must raise CCPPError mentioning 'final'.""" + text = """ + [ccpp-table-properties] + name = s + type = scheme + + [ccpp-arg-table] + name = s_finalize + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + intent = out + """ + with self.assertRaises(CCPPError) as cm: + _parse_text(text) + self.assertIn('final', str(cm.exception)) + + def test_duplicate_standard_name_in_section(self): + """Two variables with the same standard name in one section must raise.""" + text = """ + [ccpp-table-properties] + name = s + type = scheme + + [ccpp-arg-table] + name = s_run + type = scheme + [ a ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + intent = in + [ b ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + intent = in + """ + with self.assertRaises(CCPPError): + _parse_text(text) + + def test_missing_intent_for_scheme_variable(self): + """Scheme variables without ``intent`` must raise at validation time.""" + text = """ + [ccpp-table-properties] + name = s + type = scheme + + [ccpp-arg-table] + name = s_run + type = scheme + [ im ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + """ + with self.assertRaises(CCPPError): + _parse_text(text) + + def test_invalid_intent_value(self): + text = """ + [ccpp-table-properties] + name = s + type = scheme + + [ccpp-arg-table] + name = s_run + type = scheme + [ im ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + intent = banana + """ + with self.assertRaises(CCPPError): + _parse_text(text) + + def test_bad_dimensions(self): + """Dimensions not enclosed in parentheses must raise.""" + text = """ + [ccpp-table-properties] + name = h + type = host + + [ccpp-arg-table] + name = h + type = host + [ x ] + standard_name = foo + units = 1 + dimensions = no_parens + type = integer + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_missing_required_attribute(self): + """Missing ``type`` must raise at variable validation.""" + text = """ + [ccpp-table-properties] + name = h + type = host + + [ccpp-arg-table] + name = h + type = host + [ x ] + standard_name = foo + units = 1 + dimensions = () + """ + with self.assertRaises(CCPPError): + _parse_text(text) + + def test_missing_units_defaults_to_none(self): + """Omitting ``units`` is accepted; the variable's units default to ``'none'``.""" + text = """ + [ccpp-table-properties] + name = h + type = host + + [ccpp-arg-table] + name = h + type = host + [ x ] + standard_name = foo + dimensions = () + type = integer + """ + tables = _parse_text(text) + self.assertEqual(len(tables), 1) + section = tables[0].sections()[0] + var = section.variables[0] + self.assertEqual(var.units, 'none') + + def test_section_type_mismatch_raises(self): + """Section type different from table type must raise.""" + text = """ + [ccpp-table-properties] + name = h + type = host + + [ccpp-arg-table] + name = h + type = scheme + """ + with self.assertRaises(CCPPError): + _parse_text(text) + + def test_singleton_table_second_section_raises(self): + """A ``host`` table with two sections must raise.""" + text = """ + [ccpp-table-properties] + name = h + type = host + + [ccpp-arg-table] + name = h + type = host + + [ccpp-arg-table] + name = h + type = host + """ + with self.assertRaises(CCPPError): + _parse_text(text) + + +######################################################################## +# Attribute ownership enforcement tests +######################################################################## + +class TestAttributeOwnership(unittest.TestCase): + """Parse-time rejection of active/optional in wrong table types.""" + + def test_active_in_scheme_raises(self): + """'active' attribute in scheme metadata must raise ParseSyntaxError.""" + text = """ + [ccpp-table-properties] + name = my_scheme + type = scheme + + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + active = .true. + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_optional_in_host_raises(self): + """'optional' attribute in host metadata must raise ParseSyntaxError.""" + text = """ + [ccpp-table-properties] + name = my_host + type = host + + [ccpp-arg-table] + name = my_host + type = host + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + optional = true + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_optional_in_control_raises(self): + """'optional' attribute in control metadata must raise ParseSyntaxError.""" + text = """ + [ccpp-table-properties] + name = my_ctrl + type = control + + [ccpp-arg-table] + name = my_ctrl + type = control + [ x ] + standard_name = foo + units = 1 + dimensions = () + type = integer + optional = true + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_optional_in_ddt_raises(self): + """'optional' attribute in ddt metadata must raise ParseSyntaxError.""" + text = """ + [ccpp-table-properties] + name = my_ddt_type + type = ddt + + [ccpp-arg-table] + name = my_ddt_type + type = ddt + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + optional = false + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_active_in_host_allowed(self): + """'active' attribute in host metadata is valid.""" + text = """ + [ccpp-table-properties] + name = my_host + type = host + + [ccpp-arg-table] + name = my_host + type = host + [ flag ] + standard_name = my_flag + units = flag + dimensions = () + type = logical + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + active = my_flag + """ + tables = _parse_text(text) + self.assertEqual(len(tables), 1) + + def test_active_expression_is_lowercased(self): + """Mixed-case identifiers in 'active' must be normalised to lowercase + so they match the canonical lowercase standard names stored on host + variables (see check_cf_standard_name). + """ + text = """ + [ccpp-table-properties] + name = my_host + type = host + + [ccpp-arg-table] + name = my_host + type = host + [ flag ] + standard_name = flag_for_aerosol_input_mg_radiation + units = flag + dimensions = () + type = logical + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + active = (flag_for_aerosol_input_MG_radiation) + """ + tables = _parse_text(text) + x_var = next( + v for v in tables[0].sections()[0].variables + if v.local_name == 'x' + ) + self.assertEqual(x_var.active, '(flag_for_aerosol_input_mg_radiation)') + + def test_intent_in_host_raises(self): + """'intent' on a host table is now rejected (scheme-only).""" + text = """ + [ccpp-table-properties] + name = my_host + type = host + + [ccpp-arg-table] + name = my_host + type = host + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + intent = in + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_intent_in_control_raises(self): + """'intent' on a control table is now rejected (scheme-only).""" + text = """ + [ccpp-table-properties] + name = my_ctrl + type = control + + [ccpp-arg-table] + name = my_ctrl + type = control + [ x ] + standard_name = foo + units = 1 + dimensions = () + type = integer + intent = in + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_intent_in_ddt_raises(self): + """'intent' on a ddt table is rejected (scheme-only).""" + text = """ + [ccpp-table-properties] + name = my_ddt_type + type = ddt + + [ccpp-arg-table] + name = my_ddt_type + type = ddt + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + intent = inout + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_active_in_control_raises(self): + """'active' on a control table is rejected — control vars are + unconditionally framework-injected, no active expression makes + sense for them.""" + text = """ + [ccpp-table-properties] + name = my_ctrl + type = control + + [ccpp-arg-table] + name = my_ctrl + type = control + [ x ] + standard_name = foo + units = 1 + dimensions = () + type = integer + active = my_flag + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_active_in_ddt_allowed(self): + """'active' on a ddt table is allowed — a host DDT component is a + valid origin for the active-conditional storage contract.""" + text = """ + [ccpp-table-properties] + name = my_ddt_type + type = ddt + + [ccpp-arg-table] + name = my_ddt_type + type = ddt + [ flag ] + standard_name = my_flag + units = flag + dimensions = () + type = logical + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + active = my_flag + """ + tables = _parse_text(text) + self.assertEqual(len(tables), 1) + + def test_optional_in_scheme_allowed(self): + """'optional' attribute in scheme metadata is valid.""" + text = """ + [ccpp-table-properties] + name = my_scheme + type = scheme + + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + intent = in + optional = true + """ + tables = _parse_text(text) + self.assertEqual(len(tables), 1) + + def test_constituent_in_host_raises(self): + """'constituent' attribute in host metadata must raise.""" + text = """ + [ccpp-table-properties] + name = my_host + type = host + + [ccpp-arg-table] + name = my_host + type = host + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + constituent = true + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_advected_in_host_raises(self): + """'advected' attribute in host metadata must raise.""" + text = """ + [ccpp-table-properties] + name = my_host + type = host + + [ccpp-arg-table] + name = my_host + type = host + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + advected = .true. + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_molar_mass_in_ddt_raises(self): + """'molar_mass' attribute in ddt metadata must raise.""" + text = """ + [ccpp-table-properties] + name = my_ddt_type + type = ddt + + [ccpp-arg-table] + name = my_ddt_type + type = ddt + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + molar_mass = 18.0 + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + +######################################################################## +# Constituent attribute tests (scheme metadata only) +######################################################################## + +class TestConstituentAttributes(unittest.TestCase): + """Parsing and is_constituent rollup for scheme-only constituent hints.""" + + def _scheme_var(self, *, extra_attrs: str = '') -> MetaVar: + text = """ + [ccpp-table-properties] + name = my_scheme + type = scheme + + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ x ] + standard_name = foo + units = kg kg-1 + dimensions = () + type = real + intent = inout + {extra} + """.format(extra=extra_attrs) + tables = _parse_text(text) + return tables[0].sections()[0].variables[0] + + def test_constituent_true_accepted(self): + var = self._scheme_var(extra_attrs='constituent = True') + self.assertTrue(var.constituent) + self.assertTrue(var.is_constituent) + + def test_advected_dot_true_accepted(self): + var = self._scheme_var(extra_attrs='advected = .true.') + self.assertTrue(var.advected) + self.assertTrue(var.is_constituent) + + def test_molar_mass_accepted(self): + var = self._scheme_var(extra_attrs='molar_mass = 18.0') + self.assertEqual(var.molar_mass, 18.0) + self.assertTrue(var.is_constituent) + + def test_defaults_make_non_constituent(self): + var = self._scheme_var() + self.assertFalse(var.constituent) + self.assertFalse(var.advected) + self.assertEqual(var.molar_mass, 0.0) + self.assertFalse(var.is_constituent) + + def test_negative_molar_mass_rejected(self): + with self.assertRaises((CCPPError, ParseSyntaxError)): + self._scheme_var(extra_attrs='molar_mass = -1.0') + + +######################################################################## +# File-based tests (use sample_files/) +######################################################################## + +class TestParseMetadataFiles(unittest.TestCase): + """Integration tests using real ``.meta`` files in ``sample_files/``.""" + + def _sample(self, name): + return os.path.join(_SAMPLE_DIR, name) + + # ---- valid files ------------------------------------------------------- + + def test_host_simple_file(self): + tables = parse_metadata_file(self._sample('host_simple.meta')) + self.assertEqual(len(tables), 1) + tbl = tables[0] + self.assertEqual(tbl.table_name, 'physics_data') + self.assertEqual(tbl.table_type, 'host') + snames = {v.standard_name for v in tbl.variables()} + self.assertIn('horizontal_dimension', snames) + self.assertIn('vertical_layer_dimension', snames) + # loop bounds and error vars are control vars (control_simple.meta), not host vars + self.assertNotIn('horizontal_loop_begin', snames) + self.assertNotIn('horizontal_loop_end', snames) + + def test_control_simple_file(self): + tables = parse_metadata_file(self._sample('control_simple.meta')) + self.assertEqual(len(tables), 1) + self.assertEqual(tables[0].table_type, 'control') + + def test_ddt_simple_file(self): + tables = parse_metadata_file(self._sample('ddt_simple.meta')) + self.assertEqual(len(tables), 1) + tbl = tables[0] + self.assertEqual(tbl.table_type, 'ddt') + snames = [v.standard_name for v in tbl.variables()] + self.assertIn('geopotential_at_interface', snames) + self.assertIn('geopotential', snames) + # DDT field variables have no intent + for var in tbl.variables(): + self.assertIsNone(var.intent) + + def test_scheme_multipart_file(self): + """Scheme with init, run, final phases from sample file.""" + tables = parse_metadata_file(self._sample('scheme_multipart.meta')) + self.assertEqual(len(tables), 1) + tbl = tables[0] + self.assertTrue(tbl.is_scheme) + phases = {sec.phase for sec in tbl.sections()} + self.assertEqual(phases, {'init', 'run', 'final'}) + run_sec = tbl.section_for_phase('run') + self.assertIsNotNone(run_sec) + run_stdnames = {v.standard_name for v in run_sec.variables} + self.assertIn('air_temperature', run_stdnames) + + def test_host_with_ddt_instance_file(self): + """Host table declaring a DDT instance variable.""" + tables = parse_metadata_file(self._sample('host_with_ddt_instance.meta')) + self.assertEqual(len(tables), 1) + var = tables[0].sections()[0].variables[0] + self.assertEqual(var.local_name, 'gfs_statein') + self.assertEqual(var.type, 'gfs_statein_type') + self.assertEqual(var.dimensions, ['number_of_instances']) + + # ---- error files ------------------------------------------------------- + + def test_bad_module_type_file(self): + """``type = module`` must raise with a message mentioning 'host'.""" + with self.assertRaises(CCPPError) as cm: + parse_metadata_file(self._sample('bad_module_type.meta')) + self.assertIn('host', str(cm.exception).lower()) + + def test_bad_finalize_phase_file(self): + """``_finalize`` phase must raise with a message mentioning 'final'.""" + with self.assertRaises(CCPPError) as cm: + parse_metadata_file(self._sample('bad_finalize_phase.meta')) + self.assertIn('final', str(cm.exception)) + + def test_bad_invalid_type_file(self): + """Completely invalid table type raises CCPPError.""" + with self.assertRaises(CCPPError): + parse_metadata_file(self._sample('bad_invalid_type.meta')) + + def test_bad_duplicate_stdname_file(self): + """Duplicate standard name in one section raises CCPPError.""" + with self.assertRaises(CCPPError): + parse_metadata_file(self._sample('bad_duplicate_stdname.meta')) + + def test_nonexistent_file(self): + """Parsing a non-existent file raises CCPPError.""" + with self.assertRaises(CCPPError): + parse_metadata_file('/nonexistent/path/file.meta') + + +######################################################################## +# Variables() de-duplication across scheme phases +######################################################################## + +class TestTableVariables(unittest.TestCase): + """Tests for :meth:`MetadataTable.variables` cross-phase de-duplication.""" + + def test_dedup_across_phases(self): + """``ccpp_error_message`` appears in init and run; variables() returns it once.""" + text = """ + [ccpp-table-properties] + name = sch + type = scheme + + [ccpp-arg-table] + name = sch_init + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + intent = out + + [ccpp-arg-table] + name = sch_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + intent = out + [ im ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + intent = in + """ + tables = _parse_text(text) + all_vars = tables[0].variables() + snames = [v.standard_name for v in all_vars] + # ccpp_error_message appears in both phases but should be returned once + self.assertEqual(snames.count('ccpp_error_message'), 1) + self.assertIn('horizontal_dimension', snames) + + +######################################################################## +# CLI helper tests +######################################################################## + +class TestCLIHelpers(unittest.TestCase): + """Tests for CLI utility functions in ccpp_capgen.""" + + def setUp(self): + import importlib + import importlib.util + script = os.path.join(_PKG_ROOT, 'ccpp_capgen.py') + spec = importlib.util.spec_from_file_location('ccpp_capgen', script) + self.mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.mod) + + def test_split_file_list(self): + result = self.mod._split_file_list('a.meta, b.meta, c.meta') + self.assertEqual(result, ['a.meta', 'b.meta', 'c.meta']) + + def test_split_file_list_empty(self): + self.assertEqual(self.mod._split_file_list(''), []) + + def test_parse_kind_types_valid_iso_default(self): + result = self.mod._parse_kind_types(['kind_phys=REAL64', 'kind_dyn=REAL32']) + self.assertEqual(result, { + 'kind_phys': ('iso_fortran_env', 'REAL64'), + 'kind_dyn': ('iso_fortran_env', 'REAL32'), + }) + + def test_parse_kind_types_explicit_module(self): + result = self.mod._parse_kind_types(['kind_phys=my_kinds:kind_r8']) + self.assertEqual(result, {'kind_phys': ('my_kinds', 'kind_r8')}) + + def test_parse_kind_types_mixed(self): + result = self.mod._parse_kind_types([ + 'kind_iso=REAL64', + 'kind_host=my_kinds:kind_r4', + ]) + self.assertEqual(result, { + 'kind_iso': ('iso_fortran_env', 'REAL64'), + 'kind_host': ('my_kinds', 'kind_r4'), + }) + + def test_parse_kind_types_malformed(self): + with self.assertRaises(CCPPError): + self.mod._parse_kind_types(['bad_entry']) + + def test_parse_kind_types_duplicate(self): + with self.assertRaises(CCPPError): + self.mod._parse_kind_types(['kind_phys=REAL64', 'kind_phys=REAL32']) + + def test_parse_kind_types_non_iso_without_module_rejected(self): + """Bare non-ISO spec must error -- the default module only applies to ISO names.""" + with self.assertRaises(CCPPError) as cm: + self.mod._parse_kind_types(['kind_phys=kind_r8']) + self.assertIn('ISO_FORTRAN_ENV', str(cm.exception)) + + def test_parse_kind_types_too_many_colons_rejected(self): + with self.assertRaises(CCPPError): + self.mod._parse_kind_types(['kind_phys=mod:sub:spec']) + + def test_parse_kind_types_empty_module_rejected(self): + with self.assertRaises(CCPPError): + self.mod._parse_kind_types(['kind_phys=:REAL64']) + + # ---- _collect_metadata_kind_specs -------------------------------------- + + def _make_table_with_specs(self, file_path, name, specs): + ctx = ParseContext(0, file_path) + t = MetadataTable(name, 'scheme', file_path, ctx) + t.kind_specs = list(specs) + return t + + def test_collect_metadata_kind_specs_empty(self): + self.assertEqual(self.mod._collect_metadata_kind_specs([]), {}) + + def test_collect_metadata_kind_specs_single_table(self): + t = self._make_table_with_specs( + '/p/a.meta', 'a', + [('kind_temp', 'temp_kinds', 'temp_r8')], + ) + self.assertEqual( + self.mod._collect_metadata_kind_specs([t]), + {'kind_temp': ('temp_kinds', 'temp_r8')}, + ) + + def test_collect_metadata_kind_specs_identical_duplicates_collapsed(self): + spec = ('kind_temp', 'temp_kinds', 'temp_r8') + a = self._make_table_with_specs('/p/a.meta', 'a', [spec]) + b = self._make_table_with_specs('/p/b.meta', 'b', [spec]) + self.assertEqual( + self.mod._collect_metadata_kind_specs([a, b]), + {'kind_temp': ('temp_kinds', 'temp_r8')}, + ) + + def test_collect_metadata_kind_specs_conflict_raises(self): + a = self._make_table_with_specs( + '/p/a.meta', 'a', + [('kind_temp', 'temp_kinds', 'temp_r8')], + ) + b = self._make_table_with_specs( + '/p/b.meta', 'b', + [('kind_temp', 'other_kinds', 'r8')], + ) + with self.assertRaises(CCPPError) as cm: + self.mod._collect_metadata_kind_specs([a, b]) + msg = str(cm.exception) + self.assertIn("kind 'kind_temp'", msg) + self.assertIn('/p/a.meta', msg) + self.assertIn('/p/b.meta', msg) + + def test_collect_metadata_kind_specs_multiple_distinct_kinds(self): + a = self._make_table_with_specs( + '/p/a.meta', 'a', + [('kind_temp', 'temp_kinds', 'temp_r8'), + ('kind_aux', 'aux_kinds', 'aux_r4')], + ) + self.assertEqual( + self.mod._collect_metadata_kind_specs([a]), + { + 'kind_temp': ('temp_kinds', 'temp_r8'), + 'kind_aux': ('aux_kinds', 'aux_r4'), + }, + ) + + # ---- _merge_cli_and_metadata_kinds ------------------------------------- + + def test_merge_cli_and_metadata_no_overlap(self): + cli = {'kind_phys': ('iso_fortran_env', 'REAL64')} + meta = {'kind_temp': ('temp_kinds', 'temp_r8')} + self.assertEqual( + self.mod._merge_cli_and_metadata_kinds(cli, meta), + { + 'kind_phys': ('iso_fortran_env', 'REAL64'), + 'kind_temp': ('temp_kinds', 'temp_r8'), + }, + ) + + def test_merge_cli_and_metadata_identical_collapsed(self): + cli = {'kind_temp': ('temp_kinds', 'temp_r8')} + meta = {'kind_temp': ('temp_kinds', 'temp_r8')} + self.assertEqual( + self.mod._merge_cli_and_metadata_kinds(cli, meta), + {'kind_temp': ('temp_kinds', 'temp_r8')}, + ) + + def test_merge_cli_and_metadata_conflict_raises(self): + cli = {'kind_temp': ('cli_kinds', 'r8')} + meta = {'kind_temp': ('meta_kinds', 'r8')} + with self.assertRaises(CCPPError) as cm: + self.mod._merge_cli_and_metadata_kinds(cli, meta) + self.assertIn("kind 'kind_temp'", str(cm.exception).lower() + .replace('Kind', 'kind')) + self.assertIn('cli_kinds', str(cm.exception)) + self.assertIn('meta_kinds', str(cm.exception)) + + def test_merge_then_default_kind_phys_injected_when_neither_provides(self): + """Default kind_phys is injected after the merge when neither side declares it.""" + import logging + log = logging.getLogger('ccpp_capgen_test') + merged = self.mod._merge_cli_and_metadata_kinds({}, {}) + merged = self.mod._ensure_kind_phys_default(merged, log) + self.assertEqual( + merged, {'kind_phys': ('iso_fortran_env', 'REAL64')} + ) + + def test_metadata_kind_phys_suppresses_default(self): + """Metadata declaring kind_phys keeps the default from being injected.""" + import logging + log = logging.getLogger('ccpp_capgen_test') + meta = {'kind_phys': ('host_kinds', 'kind_r8')} + merged = self.mod._merge_cli_and_metadata_kinds({}, meta) + merged = self.mod._ensure_kind_phys_default(merged, log) + self.assertEqual(merged, {'kind_phys': ('host_kinds', 'kind_r8')}) + + +######################################################################## +# Tests: apply_table_props (source_path, dependencies, dependencies_path) +######################################################################## + +class TestApplyTableProps(unittest.TestCase): + """Tests for MetadataTable.apply_table_props and parsing of table-level props.""" + + def _make_table(self, file_path='/project/src/foo.meta'): + ctx = ParseContext(0, file_path) + t = MetadataTable('foo', 'scheme', file_path, ctx) + return t + + def test_defaults_when_no_props(self): + t = self._make_table() + t.apply_table_props({}) + self.assertEqual(t.dependencies, []) + # source_path defaults to the meta file's directory + self.assertEqual(t.source_path, os.path.dirname(os.path.abspath(t.file_path))) + + def test_source_path_resolved(self): + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({'source_path': 'fortran'}) + self.assertEqual(t.source_path, '/project/src/fortran') + + def test_source_path_with_dotdot(self): + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({'source_path': '../other'}) + self.assertEqual(t.source_path, '/project/other') + + def test_single_dependency_no_dep_path(self): + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({'dependencies': 'util.F90'}) + self.assertEqual(t.dependencies, ['/project/src/util.F90']) + + def test_multiple_dependencies(self): + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({'dependencies': 'a.F90, b.F90'}) + self.assertIn('/project/src/a.F90', t.dependencies) + self.assertIn('/project/src/b.F90', t.dependencies) + self.assertEqual(len(t.dependencies), 2) + + def test_dependencies_path_as_base(self): + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({'dependencies': 'util.F90', 'dependencies_path': 'lib'}) + self.assertEqual(t.dependencies, ['/project/src/lib/util.F90']) + + def test_dependencies_none_string(self): + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({'dependencies': 'none'}) + self.assertEqual(t.dependencies, []) + + def test_dependency_relative_dotdot(self): + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({'dependencies': '../../shared/helper.F90'}) + self.assertEqual(t.dependencies, ['/shared/helper.F90']) + + def test_dependencies_list_form_accumulates(self): + """When ``dependencies`` appears more than once in a single + table header, the parser passes a list to apply_table_props; + each entry can itself be a comma-separated list of paths.""" + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({'dependencies': [ + 'a.F90', + 'b.F90, c.F90', + 'sub/d.F90', + ]}) + self.assertEqual(t.dependencies, [ + '/project/src/a.F90', + '/project/src/b.F90', + '/project/src/c.F90', + '/project/src/sub/d.F90', + ]) + + def test_dependencies_list_form_honors_dep_path(self): + """``dependencies_path`` applies to every entry in a list, + whether the entry is a single path or a comma-separated + bundle.""" + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({ + 'dependencies_path': '../../', + 'dependencies': [ + 'tools/mpiutil.F90', + 'Radiation/RRTMG/radlw_main.F90,Radiation/RRTMG/radsw_main.F90', + ], + }) + self.assertEqual(t.dependencies, [ + '/tools/mpiutil.F90', + '/Radiation/RRTMG/radlw_main.F90', + '/Radiation/RRTMG/radsw_main.F90', + ]) + + def test_dependencies_list_form_with_none_skip(self): + """A ``none`` entry inside a list is silently skipped (matches + the single-string ``none`` shorthand).""" + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({'dependencies': [ + 'a.F90', + 'none', + 'b.F90', + ]}) + self.assertEqual(t.dependencies, [ + '/project/src/a.F90', + '/project/src/b.F90', + ]) + + def test_unrecognised_props_silently_ignored(self): + t = self._make_table() + t.apply_table_props({'unknown_key': 'value'}) + self.assertEqual(t.dependencies, []) + self.assertEqual(t.kind_specs, []) + + def test_kind_spec_explicit_form(self): + t = self._make_table() + t.apply_table_props({'kind_spec': 'temp_kinds:kind_temp=>temp_r8'}) + self.assertEqual( + t.kind_specs, [('kind_temp', 'temp_kinds', 'temp_r8')] + ) + + def test_kind_spec_shorthand_form(self): + t = self._make_table() + t.apply_table_props({'kind_spec': 'host_kinds:kind_r8'}) + self.assertEqual( + t.kind_specs, [('kind_r8', 'host_kinds', 'kind_r8')] + ) + + def test_kind_spec_list_accumulates(self): + t = self._make_table() + t.apply_table_props({'kind_spec': [ + 'temp_kinds:kind_temp=>temp_r8', + 'host_kinds:kind_r4', + ]}) + self.assertEqual(t.kind_specs, [ + ('kind_temp', 'temp_kinds', 'temp_r8'), + ('kind_r4', 'host_kinds', 'kind_r4'), + ]) + + def test_kind_spec_malformed_raises(self): + t = self._make_table() + with self.assertRaises(CCPPError): + t.apply_table_props({'kind_spec': 'real8'}) + + def test_kind_spec_extra_arrow_segment_rejected(self): + t = self._make_table() + with self.assertRaises(CCPPError): + t.apply_table_props( + {'kind_spec': 'mod:a=>b=>c'} + ) + + def test_module_name_default_empty(self): + t = self._make_table() + t.apply_table_props({}) + self.assertEqual(t.module_name, '') + + def test_module_name_explicit(self): + """``module_name`` override for cases where the Fortran module + name differs from the metadata table name (e.g. ``effr_pre`` table + whose Fortran module is ``mod_effr_pre``).""" + t = self._make_table() + t.apply_table_props({'module_name': 'mod_effr_pre'}) + self.assertEqual(t.module_name, 'mod_effr_pre') + + def test_module_name_whitespace_stripped(self): + t = self._make_table() + t.apply_table_props({'module_name': ' mod_foo '}) + self.assertEqual(t.module_name, 'mod_foo') + + def test_module_name_empty_string_keeps_default(self): + t = self._make_table() + t.apply_table_props({'module_name': ' '}) + self.assertEqual(t.module_name, '') + + +class TestTablePropsParseIntegration(unittest.TestCase): + """Verify that source_path/dependencies are parsed from actual meta text.""" + + def _parse(self, src, fname='/project/src/my_scheme.meta'): + return _parse_lines(src.splitlines(keepends=True), fname) + + def test_source_path_parsed(self): + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + source_path = fortran + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """) + tbls = self._parse(src) + self.assertEqual(len(tbls), 1) + self.assertEqual(tbls[0].source_path, '/project/src/fortran') + + def test_dependencies_parsed(self): + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + dependencies = util.F90, helper.F90 + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """) + tbls = self._parse(src) + self.assertIn('/project/src/util.F90', tbls[0].dependencies) + self.assertIn('/project/src/helper.F90', tbls[0].dependencies) + + def test_dependencies_path_applied(self): + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + dependencies = qux.F90 + dependencies_path = adjust + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """) + tbls = self._parse(src) + self.assertEqual(tbls[0].dependencies, ['/project/src/adjust/qux.F90']) + + def test_dependencies_repeated_in_header(self): + """``dependencies`` may appear multiple times in a single table + header — each line contributes its (comma-separated) paths. + Real-world example from CCPP physics: dependencies_path = ../../ + followed by ~7 dependencies lines that each list a comma- + separated bundle in a different subtree. + """ + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = GFS_rrtmg_setup + type = scheme + dependencies_path = ../../ + dependencies = tools/mpiutil.F90 + dependencies = hooks/machine.F + dependencies = Radiation/radiation_aerosols.f + dependencies = Radiation/radiation_astronomy.f, Radiation/radiation_clouds.f, Radiation/radiation_gases.f + dependencies = Radiation/RRTMG/radlw_main.F90,Radiation/RRTMG/radlw_param.f,Radiation/RRTMG/radsw_main.F90,Radiation/RRTMG/radsw_param.f + [ccpp-arg-table] + name = GFS_rrtmg_setup_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """) + tbls = self._parse(src, fname='/project/physics/src/GFS_rrtmg_setup.meta') + self.assertEqual(len(tbls), 1) + # dependencies_path = ../../ → base is /project/ + self.assertEqual(tbls[0].dependencies, [ + '/project/tools/mpiutil.F90', + '/project/hooks/machine.F', + '/project/Radiation/radiation_aerosols.f', + '/project/Radiation/radiation_astronomy.f', + '/project/Radiation/radiation_clouds.f', + '/project/Radiation/radiation_gases.f', + '/project/Radiation/RRTMG/radlw_main.F90', + '/project/Radiation/RRTMG/radlw_param.f', + '/project/Radiation/RRTMG/radsw_main.F90', + '/project/Radiation/RRTMG/radsw_param.f', + ]) + + def test_dependencies_path_still_rejects_duplicates(self): + """Even with ``dependencies`` now accumulating, the + ``dependencies_path`` attribute itself remains single-valued — + repeating it is a metadata error.""" + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + dependencies_path = ../ + dependencies_path = ../../ + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """) + with self.assertRaises(CCPPError): + self._parse(src) + + def test_source_path_still_rejects_duplicates(self): + """``source_path`` is single-valued too.""" + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + source_path = fortran + source_path = src + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """) + with self.assertRaises(CCPPError): + self._parse(src) + + def test_kind_spec_single_line_parsed(self): + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + kind_spec = temp_kinds:kind_temp=>temp_r8 + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """) + tbls = self._parse(src) + self.assertEqual( + tbls[0].kind_specs, + [('kind_temp', 'temp_kinds', 'temp_r8')], + ) + + def test_kind_spec_multiple_lines_accumulate(self): + """Repeat ``kind_spec`` lines accumulate without a duplicate-key error.""" + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + kind_spec = temp_kinds:kind_temp=>temp_r8 + kind_spec = host_kinds:kind_r4 + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """) + tbls = self._parse(src) + self.assertEqual(tbls[0].kind_specs, [ + ('kind_temp', 'temp_kinds', 'temp_r8'), + ('kind_r4', 'host_kinds', 'kind_r4'), + ]) + + def test_kind_spec_malformed_value_raises(self): + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + kind_spec = not_a_kind_spec + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """) + with self.assertRaises(CCPPError): + self._parse(src) + + def test_props_at_eof_no_arg_table(self): + """source_path parsed even if there are no [ccpp-arg-table] sections.""" + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + source_path = src + """) + tbls = self._parse(src) + self.assertEqual(len(tbls), 1) + self.assertEqual(tbls[0].source_path, '/project/src/src') + + def test_multiple_tables_independent_deps(self): + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = scheme_a + type = scheme + dependencies = a.F90 + [ccpp-table-properties] + name = scheme_b + type = scheme + dependencies = b.F90 + """) + tbls = self._parse(src) + self.assertEqual(len(tbls), 2) + self.assertEqual(tbls[0].dependencies, ['/project/src/a.F90']) + self.assertEqual(tbls[1].dependencies, ['/project/src/b.F90']) + + +######################################################################## +# Doctest loader +######################################################################## + +def load_tests(loader, tests, ignore): + """Auto-discover and run all doctests in the metadata subpackage.""" + import doctest + import metadata.metadata_table as _mt + import metadata.parse_tools.parse_source as _ps + import metadata.parse_tools.parse_log as _pl + tests.addTests(doctest.DocTestSuite(_mt)) + tests.addTests(doctest.DocTestSuite(_ps)) + tests.addTests(doctest.DocTestSuite(_pl)) + return tests + + +######################################################################## + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/test_registered_dimensions.py b/unit-tests/test_registered_dimensions.py new file mode 100644 index 00000000..73a77c5e --- /dev/null +++ b/unit-tests/test_registered_dimensions.py @@ -0,0 +1,104 @@ +"""Tests for :mod:`metadata.registered_dimensions`. + +This module is the single source of truth for capgen's registered +scalar-index dimension table. Tests here cover: + + * Every entry in :data:`SCALAR_INDEX_DIMS` is present (regression + against accidental removal during refactors). + * Helper functions return the expected values for entries inside and + outside the table. + * Adding a new pairing follows a clear, documented recipe — the + test_table_shape test exists so anyone extending the dict sees what + they need to update. + +When you add a new dim → index pair, add a corresponding test below +following the existing pattern. See the module's top docstring for +the four-step extension recipe. +""" + +import unittest + +from metadata.registered_dimensions import ( + SCALAR_INDEX_DIMS, + is_scalar_index_dim, + registered_count_dims, + registered_index_vars, + scalar_index_for, +) + + +class TestSCalarIndexDimsContents(unittest.TestCase): + """The registered table must contain at least the two pairings that + capgen has committed to. Removing or renaming either is a + breaking change for hosts in production.""" + + def test_number_of_instances_pair(self): + self.assertEqual( + SCALAR_INDEX_DIMS['number_of_instances'], + 'instance_number', + ) + + def test_number_of_threads_pair(self): + self.assertEqual( + SCALAR_INDEX_DIMS['number_of_threads'], + 'thread_number', + ) + + def test_no_dead_instance_dimension_entry(self): + """The pre-2026-05-13 'instance_dimension' name was never used + in any real metadata; it shipped only in the original design + prompt. It MUST stay out of the registered table.""" + self.assertNotIn('instance_dimension', SCALAR_INDEX_DIMS) + + +class TestHelpers(unittest.TestCase): + + def test_scalar_index_for_registered(self): + self.assertEqual(scalar_index_for('number_of_instances'), + 'instance_number') + self.assertEqual(scalar_index_for('number_of_threads'), + 'thread_number') + + def test_scalar_index_for_unregistered_returns_none(self): + self.assertIsNone(scalar_index_for('horizontal_dimension')) + self.assertIsNone(scalar_index_for('vertical_layer_dimension')) + self.assertIsNone(scalar_index_for('number_of_ccpp_constituents')) + + def test_is_scalar_index_dim(self): + self.assertTrue(is_scalar_index_dim('number_of_instances')) + self.assertTrue(is_scalar_index_dim('number_of_threads')) + self.assertFalse(is_scalar_index_dim('horizontal_dimension')) + self.assertFalse(is_scalar_index_dim('instance_dimension')) + + def test_registered_count_dims_returns_keys(self): + self.assertEqual(registered_count_dims(), + frozenset(SCALAR_INDEX_DIMS)) + + def test_registered_index_vars_returns_values(self): + self.assertEqual(registered_index_vars(), + frozenset(SCALAR_INDEX_DIMS.values())) + + +class TestExtensionRecipe(unittest.TestCase): + """Anyone extending SCALAR_INDEX_DIMS should: + 1. Add an entry. + 2. Add a regression test in TestSCalarIndexDimsContents above. + 3. Update doc/migration.md §3 and doc/redesign_prompt.md §4.3. + 4. Add a unit test exercising the new pairing in the resolver. + This single test exists so the table's shape is asserted in one + place — if you add a key, this test fails and points you at the + above checklist. + """ + + def test_table_size(self): + # Update this assertion when you add a new pairing. See the + # docstring on this class for the full checklist. + self.assertEqual( + len(SCALAR_INDEX_DIMS), 2, + "Registered scalar-index table grew without test update — " + "see TestExtensionRecipe checklist", + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/test_suite_cap.py b/unit-tests/test_suite_cap.py new file mode 100644 index 00000000..b852d034 --- /dev/null +++ b/unit-tests/test_suite_cap.py @@ -0,0 +1,611 @@ +"""Unit tests for generator.suite_cap.""" + +import doctest +import os +import tempfile +import unittest +from unittest.mock import MagicMock + +from metadata.metadata_table import parse_metadata_file +from metadata.variable_resolver import build_flat_host_dict, SchemeStore +from generator.suite_resolver import resolve_suite, ResolvedGroup +from generator.suite_cap import ( + _all_suite_scheme_names, + _schemes_with_register, + _suite_ctrl_args_for_phase, + _generate_suite_cap, + write_suite_cap, +) +from test_suite_resolver import ( + _load_full_host_dict, + _load_scheme_store, + _parse_suite, + _parse, + _sf, +) + + +def _resolve(): + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + return resolve_suite(suite, store, hd), store + + +def _generate(): + suite_resolution, store = _resolve() + return _generate_suite_cap('test_simple', suite_resolution, store) + + +class TestAllSuiteSchemeNames(unittest.TestCase): + + def test_single_scheme(self): + suite_resolution, _ = _resolve() + names = _all_suite_scheme_names(suite_resolution) + self.assertIn('temp_calc_adjust', names) + + def test_no_duplicates(self): + suite_resolution, _ = _resolve() + names = _all_suite_scheme_names(suite_resolution) + self.assertEqual(len(names), len(set(names))) + + +class TestSchemesWithRegister(unittest.TestCase): + + def test_none_have_register(self): + suite_resolution, store = _resolve() + names = _all_suite_scheme_names(suite_resolution) + reg = _schemes_with_register(names, store) + # temp_calc_adjust has no register phase. + self.assertEqual(reg, []) + + def test_scheme_with_register(self): + store = MagicMock() + store.phases_for.side_effect = ( + lambda n: ['register', 'run'] if n == 'my_scheme' else ['run'] + ) + result = _schemes_with_register(['my_scheme', 'other_scheme'], store) + self.assertEqual(result, ['my_scheme']) + + +class TestSuiteCtrlArgsForPhase(unittest.TestCase): + + def test_only_error_ctrl_args_in_test_case(self): + suite_resolution, _ = _resolve() + # temp_calc_adjust uses errmsg/errflg which are now control vars. + args = _suite_ctrl_args_for_phase(suite_resolution, 'run') + std_names = {a.standard_name for a in args} + self.assertEqual(std_names, {'ccpp_error_message', 'ccpp_error_code'}) + + def test_unknown_phase_returns_empty(self): + suite_resolution, _ = _resolve() + args = _suite_ctrl_args_for_phase(suite_resolution, 'register') + self.assertEqual(args, []) + + +class TestGenerateSuiteCapModule(unittest.TestCase): + """Suite cap with no register-providing schemes: constituent USE and + _register are NOT emitted (conditional emission).""" + + def setUp(self): + self.lines = _generate() + self.text = '\n'.join(self.lines) + + def test_module_header_comment(self): + self.assertTrue(self.lines[0].startswith('!')) + self.assertIn('test_simple', self.lines[0]) + + def test_module_declaration(self): + self.assertIn('module ccpp_test_simple_cap', self.text) + self.assertIn('end module ccpp_test_simple_cap', self.text) + + def test_does_not_use_constituent_mod(self): + # No register-providing schemes → no constituent module dependency. + self.assertNotIn('use ccpp_constituent_prop_mod', self.text) + self.assertNotIn('ccpp_model_constituents_t', self.text) + + def test_uses_group_cap_mod(self): + self.assertIn('use ccpp_test_simple_physics_cap', self.text) + + def test_implicit_none_private(self): + self.assertIn('implicit none', self.text) + self.assertIn('private', self.text) + + def test_public_register_always_emitted(self): + # _register is mandatory in the new design — always public. + self.assertIn('public :: test_simple_register', self.text) + + def test_public_init_final(self): + self.assertIn('public :: test_simple_init', self.text) + self.assertIn('public :: test_simple_final', self.text) + + def test_public_all_physics_phases(self): + for phase in ('init', 'timestep_init', 'run', 'timestep_final', 'final'): + self.assertIn( + 'public :: test_simple_physics_{}'.format(phase), self.text + ) + + def test_contains_block(self): + self.assertIn('contains', self.lines) + + +class TestRegisterSubroutineAlwaysEmitted(unittest.TestCase): + """``_register`` is mandatory and emitted unconditionally. When no + schemes have a register phase, the body is just the state-alloc + guard + + state-transition skeleton; no scheme calls.""" + + def setUp(self): + lines = _generate() + self.text = '\n'.join(lines) + + def test_register_subroutine_present(self): + self.assertIn('subroutine test_simple_register', self.text) + self.assertIn('end subroutine test_simple_register', self.text) + + def test_no_constituents_arg(self): + # Constituents are now opt-in via type=host; not in the cap at all + # when no register-phase scheme declares ccpp_constituent_properties_t. + self.assertNotIn('constituents', self.text) + self.assertNotIn('ccpp_constituent_prop_mod', self.text) + + def test_no_scheme_register_calls(self): + # temp_calc_adjust has no register phase → no scheme_register call. + self.assertNotIn('call temp_calc_adjust_register', self.text) + + def test_state_alloc_called(self): + # Register always allocates state (idempotent) on first call. + self.assertIn('call test_simple_suite_state_alloc', self.text) + + def test_idempotent_guard(self): + # Per-instance idempotent skip if already at REGISTERED or beyond. + self.assertIn('>= CCPP_SUITE_REGISTERED', self.text) + + def test_state_transition(self): + self.assertIn('= CCPP_SUITE_REGISTERED', self.text) + + +class TestInitFinalSubroutines(unittest.TestCase): + + def setUp(self): + self.text = '\n'.join(_generate()) + + def test_init_subroutine(self): + self.assertIn('subroutine test_simple_init(errmsg, errflg)', self.text) + self.assertIn('end subroutine test_simple_init', self.text) + + def test_final_subroutine(self): + self.assertIn('subroutine test_simple_final(errmsg, errflg)', self.text) + self.assertIn('end subroutine test_simple_final', self.text) + + def test_init_calls_group_state_alloc(self): + # No host_dict passed → single-instance → literal 1 for ninstances. + self.assertIn( + 'call physics_state_alloc(1, errmsg, errflg)', + self.text, + ) + + def test_register_calls_suite_state_alloc(self): + # Suite state allocation happens in _register, not _init. + # No host_dict → single-instance → literal 1 for ninstances. + self.assertIn( + 'call test_simple_suite_state_alloc(1, errmsg, errflg)', + self.text, + ) + + def test_final_calls_state_dealloc(self): + self.assertIn('call physics_state_dealloc(errmsg, errflg)', self.text) + + +class TestPhysicsDispatch(unittest.TestCase): + + def setUp(self): + self.text = '\n'.join(_generate()) + + def test_run_dispatch_present(self): + # No control vars in test setup → no-arg signature. + self.assertIn('subroutine test_simple_physics_run()', self.text) + self.assertIn('end subroutine test_simple_physics_run', self.text) + + def test_run_dispatches_to_group_cap(self): + # No group_name control var → unconditional call, no select case. + self.assertIn('call physics_run()', self.text) + + def test_init_dispatches_to_group_cap(self): + self.assertIn('call physics_init()', self.text) + + def test_final_dispatches_to_group_cap(self): + self.assertIn('call physics_final()', self.text) + + def test_timestep_init_dispatches_to_group_cap(self): + # Group phase subroutines are always emitted so the state machine + # transitions through every phase, even when no scheme has a routine + # for that phase — so the dispatch must always call into the group cap. + self.assertIn('subroutine test_simple_physics_timestep_init()', self.text) + self.assertIn('call physics_timestep_init()', self.text) + + def test_timestep_final_dispatches_to_group_cap(self): + self.assertIn('subroutine test_simple_physics_timestep_final()', self.text) + self.assertIn('call physics_timestep_final()', self.text) + + def test_no_select_case_without_group_name_ctrl(self): + # No group_name control var in test setup → no select case dispatch. + self.assertNotIn('select case(trim(group_name))', self.text) + + +class TestGroupDispatchUnknownGroupError(unittest.TestCase): + """When the host carries ``group_name`` in its control table the + suite cap dispatches via ``select case(trim(group_name))``. Every + such dispatch MUST end with a ``case default`` that sets errflg=1 + and writes a message naming the unknown group — caller asking for a + group this suite doesn't define is a runtime error, not a silent + fall-through. + """ + + def setUp(self): + suite_resolution, store = _resolve() + self.text = '\n'.join( + _generate_suite_cap('test_simple', suite_resolution, store, _load_full_host_dict()) + ) + + def _phase_block(self, phase): + sub = 'subroutine test_simple_physics_{}'.format(phase) + start = self.text.index(sub) + end = self.text.index('end subroutine test_simple_physics_{}'.format(phase), start) + return self.text[start:end] + + def test_run_dispatch_has_case_default(self): + block = self._phase_block('run') + # control_full.meta names the group_name local as grp_name; assert + # against the local name rather than the standard name. + self.assertIn('select case(trim(grp_name))', block) + self.assertIn('case default', block) + # errflg must be set non-zero in the default branch. + self.assertRegex(block, r'case default[^!]*?errflg = 1') + + def test_default_message_names_unknown_group(self): + block = self._phase_block('run') + self.assertIn( + "test_simple_physics_run: unknown group: ' // trim(grp_name)", + block, + ) + + def test_default_branch_returns(self): + """The default branch must ``return`` after setting errflg — + otherwise execution falls out of the select and into any code + that follows the dispatch (state transitions, etc.).""" + block = self._phase_block('run') + # Find the case-default section. + case_idx = block.index('case default') + end_idx = block.index('end select', case_idx) + default_block = block[case_idx:end_idx] + self.assertIn('return', default_block) + + def test_all_phases_have_default_case(self): + for phase in ('init', 'timestep_init', 'run', 'timestep_final', 'final'): + block = self._phase_block(phase) + self.assertIn('case default', block, + "phase '{}' missing case default".format(phase)) + + +class TestGroupDispatchErrorPropagation(unittest.TestCase): + """A ``group_name='all'`` dispatch must stop and return on the FIRST + group's error. Each group phase subroutine resets ``errflg=0`` on entry, + so without a guard between group calls a later group's success would mask + an earlier group's failure -- which then resurfaces downstream only as an + "invalid group state" when ``run`` finds the failed group never reached + ``IN_TIMESTEP``. (Regression: CAM-SIMA cam4 physics_before_coupler.)""" + + def _two_group_run_all_block(self): + sr, store = _resolve() + g0 = sr.groups[0] + # Synthesize a second group that shares the first's phase calls so the + # case('', 'all') path emits two group calls. + sr.groups.append(ResolvedGroup( + group_name='physics_second', + phase_calls=g0.phase_calls, + dim_uses=g0.dim_uses, + )) + text = '\n'.join( + _generate_suite_cap('test_simple', sr, store, _load_full_host_dict()) + ) + sub = 'subroutine test_simple_physics_run' + s = text.index(sub) + e = text.index('end ' + sub, s) + block = text[s:e] + a = block.index("case('', 'all')") + nxt = block.index("case('physics", a + 1) # first individual group case + return block[a:nxt] + + def test_guard_between_group_calls(self): + all_block = self._two_group_run_all_block() + # Each of the two group calls is followed by an errflg guard. + self.assertEqual( + all_block.count('if (errflg /= 0) return'), 2, all_block + ) + # The guard after the first call must precede the second call so the + # second group is unreachable once the first has failed. + first_call = all_block.index('call physics_run(') + guard = all_block.index('if (errflg /= 0) return', first_call) + second_call = all_block.index('call physics_second_run(') + self.assertLess(guard, second_call, all_block) + + +class TestWriteSuiteCap(unittest.TestCase): + + def test_writes_file(self): + suite_resolution, store = _resolve() + with tempfile.TemporaryDirectory() as tmpdir: + path = write_suite_cap('test_simple', suite_resolution, store, tmpdir) + self.assertTrue(os.path.isfile(path)) + self.assertEqual(os.path.basename(path), 'ccpp_test_simple_cap.F90') + + def test_file_content(self): + suite_resolution, store = _resolve() + with tempfile.TemporaryDirectory() as tmpdir: + path = write_suite_cap('test_simple', suite_resolution, store, tmpdir) + with open(path) as fh: + content = fh.read() + self.assertIn('module ccpp_test_simple_cap', content) + # Register subroutine is always emitted now. + self.assertIn('subroutine test_simple_register', content) + self.assertTrue(content.endswith('\n')) + + def test_creates_output_dir(self): + suite_resolution, store = _resolve() + with tempfile.TemporaryDirectory() as tmpdir: + subdir = os.path.join(tmpdir, 'caps') + write_suite_cap('test_simple', suite_resolution, store, subdir) + self.assertTrue(os.path.isdir(subdir)) + + +class TestFinalSubroutineStateMachine(unittest.TestCase): + """``_final`` transitions per-instance state to UNREGISTERED and + triggers a last-to-leave dealloc when every instance has finalized.""" + + def setUp(self): + self.text = '\n'.join(_generate()) + + def test_idempotent_unregistered_skip(self): + self.assertIn('== CCPP_SUITE_UNREGISTERED', self.text) + + def test_state_transition_to_unregistered(self): + self.assertIn('= CCPP_SUITE_UNREGISTERED', self.text) + + def test_last_to_leave_dealloc(self): + self.assertIn( + 'all(ccpp_suite_state == CCPP_SUITE_UNREGISTERED)', self.text, + ) + + +class TestSuiteCapNoConstituentDeclarations(unittest.TestCase): + """Under option A the suite cap no longer owns constituent state. + + All declarations (constituent obj, pointers, index_of_) live in + the host-wide ``ccpp_host_constituents`` module. The suite cap is + responsible only for packing per-suite dynamic-constituent arrays + into the shared buffer during ``_register``. + """ + + def setUp(self): + from test_suite_resolver import ( + _load_constituent_host_dict, + _load_constituent_consumer_store, + ) + self.hd = _load_constituent_host_dict() + self.store = _load_constituent_consumer_store() + self.suite = _parse_suite('suite_consume_constituent.xml') + self.suite_resolution = resolve_suite(self.suite, self.store, self.hd) + self.text = '\n'.join( + _generate_suite_cap('consume_consts', self.suite_resolution, self.store, self.hd) + ) + + def test_no_kind_phys_import(self): + # Suite cap doesn't need kind_phys — constituent arrays live elsewhere. + self.assertNotIn('use ccpp_kinds, only: kind_phys', self.text) + + def test_no_constituent_pointer_type_import(self): + self.assertNotIn('ccpp_constituent_prop_ptr_t', self.text) + + def test_no_module_level_pointers(self): + self.assertNotIn( + 'pointer, public :: ccpp_constituents', self.text, + ) + self.assertNotIn( + 'pointer, public :: ccpp_constituent_tendencies', self.text, + ) + + def test_no_index_of_X_in_suite_cap(self): + self.assertNotIn('index_of_cloud_liquid_water_mixing_ratio', self.text) + + def test_no_const_index_call_in_init(self): + init_body = self.text.split('subroutine consume_consts_init')[1].split( + 'end subroutine consume_consts_init' + )[0] + self.assertNotIn('%const_index(', init_body) + self.assertNotIn('%vars_layer', init_body) + + +class TestSuiteCapNoConstituentEmissionWhenAbsent(unittest.TestCase): + """When the suite does not reference any constituent state, the + suite cap emits no constituent-related code.""" + + def setUp(self): + self.text = '\n'.join(_generate()) + + def test_no_constituent_pointers(self): + self.assertNotIn('=> null()', self.text) + + def test_no_index_of_declarations(self): + self.assertNotIn('index_of_', self.text) + + def test_no_constituent_prop_ptr_type_import(self): + self.assertNotIn('ccpp_constituent_prop_ptr_t', self.text) + + +class TestTraceEmission(unittest.TestCase): + """The generated suite cap always carries a module-level ``trace`` + parameter (default .false.) and a gated ``write(error_unit,*)`` in + every physics-dispatch subroutine; ``trace=True`` flips the default. + """ + + def setUp(self): + self.suite_resolution, self.store = _resolve() + self.hd = _load_full_host_dict() + + def test_module_gate_default_off(self): + text = '\n'.join(_generate_suite_cap( + 'test_simple', self.suite_resolution, self.store, self.hd, + )) + self.assertIn('logical, parameter :: trace = .false.', text) + + def test_module_gate_default_on(self): + text = '\n'.join(_generate_suite_cap( + 'test_simple', self.suite_resolution, self.store, self.hd, + trace=True, + )) + self.assertIn('logical, parameter :: trace = .true.', text) + self.assertNotIn('logical, parameter :: trace = .false.', text) + + def test_error_unit_use_unconditional(self): + text = '\n'.join(_generate_suite_cap( + 'test_simple', self.suite_resolution, self.store, self.hd, + )) + self.assertIn( + 'use, intrinsic :: iso_fortran_env, only: error_unit', text, + ) + + def test_trace_block_present_in_physics_phases(self): + text = '\n'.join(_generate_suite_cap( + 'test_simple', self.suite_resolution, self.store, self.hd, + )) + for phase in ('init', 'timestep_init', 'run', + 'timestep_final', 'final'): + self.assertIn( + "'CCPP TRACE test_simple_physics_{}:'".format(phase), + text, + msg='trace string missing for phase {}'.format(phase), + ) + + def test_write_suite_cap_threads_trace_flag(self): + with tempfile.TemporaryDirectory() as tmpdir: + out_path = write_suite_cap( + 'test_simple', self.suite_resolution, self.store, tmpdir, + self.hd, trace=True, + ) + with open(out_path) as fh: + text = fh.read() + self.assertIn('logical, parameter :: trace = .true.', text) + + +# A register-phase scheme that consumes a HOST array sliced by a HOST +# dimension (gases(1:n_gases)) and produces a constituent (so the register +# call body is emitted). Mirrors rrtmgp_constituents_register, whose +# rad_climate(1:rad_climate_dimension) arg triggered the bug. +_REG_HOSTDIM_HOST = ''' +[ccpp-table-properties] + name = gasreg_host + type = host +[ccpp-arg-table] + name = gasreg_host + type = host +[ n_gases ] + standard_name = gas_list_dimension + units = count + dimensions = () + type = integer +[ gases ] + standard_name = list_of_gases + units = none + type = character | kind = len=256 + dimensions = (gas_list_dimension) +''' + +_REG_HOSTDIM_SCHEME = ''' +[ccpp-table-properties] + name = gas_register + type = scheme +[ccpp-arg-table] + name = gas_register_register + type = scheme +[ gases ] + standard_name = list_of_gases + units = none + type = character | kind = len=256 + dimensions = (gas_list_dimension) + intent = in +[ dyn_consts ] + standard_name = gasreg_dyn_consts + units = none + type = ccpp_constituent_properties_t + allocatable = True + dimensions = (:) + intent = out +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + +_REG_HOSTDIM_SUITE = ( + '\n' + '\n' + ' \n' + ' gas_register\n' + ' \n' + '\n' +) + + +class TestRegisterHostDimensionImported(unittest.TestCase): + """A register-phase scheme arg that is a host array sliced by a host + dimension must import BOTH the array and its dimension symbol into + _register. Regression for the rrtmgp + 'rad_climate_dimension has no IMPLICIT type' suite-cap error.""" + + @classmethod + def setUpClass(cls): + import logging + from generator.suite_xml import parse_suite_xml + host_tbls = _parse(_REG_HOSTDIM_HOST, 'gasreg_host.meta') + ctrl_tbls = parse_metadata_file(_sf('control_full.meta')) + hd = build_flat_host_dict(host_tbls, ctrl_tbls, []) + store = SchemeStore.build_from( + _parse(_REG_HOSTDIM_SCHEME, 'gas_register.meta')) + with tempfile.TemporaryDirectory() as tmp: + sx = os.path.join(tmp, 'suite_gasreg.xml') + with open(sx, 'w') as fh: + fh.write(_REG_HOSTDIM_SUITE) + suite = parse_suite_xml(sx, tmp, logging.getLogger('test'), + skip_validation=True) + sr = resolve_suite(suite, store, hd) + cls.text = '\n'.join(_generate_suite_cap('gasreg', sr, store, hd)) + cls.reg = cls.text.split('subroutine gasreg_register')[1].split( + 'end subroutine gasreg_register')[0] + + def test_call_slices_array_by_dimension(self): + # The call subscript references the host dimension's local name. + self.assertIn('gases(1:n_gases)', self.reg) + + def test_dimension_symbol_imported(self): + # Both the array and the dimension must be USE'd from the host module. + self.assertRegex(self.reg, r'use gasreg_host, only:[^\n]*\bn_gases\b') + self.assertRegex(self.reg, r'use gasreg_host, only:[^\n]*\bgases\b') + + +def load_tests(loader, tests, ignore): + import generator.suite_cap as subcycle + tests.addTests(doctest.DocTestSuite(subcycle)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_suite_data.py b/unit-tests/test_suite_data.py new file mode 100644 index 00000000..6c92cff4 --- /dev/null +++ b/unit-tests/test_suite_data.py @@ -0,0 +1,289 @@ +"""Unit tests for generator.suite_data.""" + +import doctest +import os +import tempfile +import unittest + +from metadata.parse_tools import CCPPError +from generator.suite_data import ( + _generate_suite_data, + _collect_dim_uses, + _dim_local_expr, + write_suite_data, +) +from generator.suite_resolver import SuiteVar + + +def _make_sv(std_name, local='myvar', type_='real', kind='kind_phys', + units='K', dims=None, scheme='sch', phase='run'): + return SuiteVar( + standard_name=std_name, + local_name=local, + type_=type_, + kind=kind, + units=units, + dimensions=dims or [], + source_scheme=scheme, + source_phase=phase, + ) + + +class TestGenerateSuiteDataEmpty(unittest.TestCase): + """Suite with no suite-owned variables.""" + + def setUp(self): + self.lines = _generate_suite_data('mysuite', {}) + self.text = '\n'.join(self.lines) + + def test_module_header_comment(self): + self.assertTrue(self.lines[0].startswith('!')) + self.assertIn('mysuite', self.lines[0]) + + def test_module_declaration(self): + self.assertIn('module ccpp_mysuite_data', self.text) + self.assertIn('end module ccpp_mysuite_data', self.text) + + def test_implicit_none_private(self): + self.assertIn('implicit none', self.text) + self.assertIn('private', self.text) + + def test_ddt_type_name(self): + self.assertIn('type, public :: ccpp_mysuite_data_t', self.text) + self.assertIn('end type ccpp_mysuite_data_t', self.text) + + def test_empty_ddt_comment(self): + self.assertIn('no suite-owned variables', self.text) + + def test_module_instance(self): + self.assertIn( + 'type(ccpp_mysuite_data_t), allocatable, target, public :: ccpp_suite_data(:)', + self.text, + ) + + def test_no_trailing_newlines(self): + for line in self.lines: + self.assertNotIn('\n', line) + + +class TestGenerateSuiteDataWithVars(unittest.TestCase): + """Suite with suite-owned variables.""" + + def setUp(self): + suite_vars = { + 'air_temp_adjusted': _make_sv( + 'air_temp_adjusted', 'temp_adj', 'real', 'kind_phys', 'K', + dims=['horizontal_dimension', 'vertical_layer_dimension'], + ), + 'humidity': _make_sv( + 'humidity', 'q', 'real', 'kind_phys', 'kg kg-1', + dims=['horizontal_dimension'], + ), + } + self.lines = _generate_suite_data('suite_x', suite_vars) + self.text = '\n'.join(self.lines) + + def test_fields_present(self): + self.assertIn('temp_adj', self.text) + self.assertIn('q', self.text) + + def test_allocatable_arrays(self): + # Array fields should be allocatable. + self.assertIn('allocatable', self.text) + + def test_real_kind(self): + self.assertIn('real(kind=kind_phys)', self.text) + + def test_uses_ccpp_kinds(self): + # Suite vars referencing ``kind_phys`` must USE it from ccpp_kinds. + self.assertIn('use ccpp_kinds, only: kind_phys', self.text) + + def test_no_empty_comment(self): + self.assertNotIn('no suite-owned variables', self.text) + + def test_fields_sorted(self): + # Sorted by standard_name: air_temp_adjusted before humidity. + idx_temp = self.text.index('temp_adj') + idx_q = self.text.index(' q') + self.assertLess(idx_temp, idx_q) + + def test_components_have_no_target_attribute(self): + """Fortran does NOT allow ``target`` as a derived-type component + attribute; the TARGET attribute lives on the outer instance array + instead. See :func:`test_instance_array_is_target`.""" + # No component-level ``, target`` substring should appear on a + # field declaration line. We check the strict patterns we'd + # emit if this regressed. + self.assertNotIn('allocatable, target :: temp_adj', self.text) + self.assertNotIn('allocatable, target :: q', self.text) + + def test_instance_array_is_target(self): + """The module-level instance array carries TARGET so every + ``ccpp_suite_data(i)%component(...)`` subobject is a valid + pointer-assignment target (used by the group cap to pointer-assign + optional-arg wrappers and transformation temporaries). + """ + self.assertIn( + 'type(ccpp_suite_x_data_t), allocatable, target, public :: ' + 'ccpp_suite_data(:)', + self.text, + ) + + +class TestGenerateSuiteDataScalar(unittest.TestCase): + """Suite var that is a scalar (no dimensions).""" + + def setUp(self): + suite_var = _make_sv('flag_var', 'flag', 'logical', '', '1', dims=[]) + self.lines = _generate_suite_data('s', {'flag_var': suite_var}) + self.text = '\n'.join(self.lines) + + def test_no_allocatable_for_scalar(self): + # Scalar fields should NOT be allocatable; only the outer instance array is. + type_body = self.text.split('type, public ::')[1].split('end type')[0] + self.assertNotIn('allocatable', type_body) + + def test_logical_type(self): + self.assertIn('logical', self.text) + + def test_no_ccpp_kinds_use_when_no_kinded_vars(self): + # No real(kind=...) vars → no ``use ccpp_kinds`` should be emitted. + self.assertNotIn('use ccpp_kinds', self.text) + + +class TestGenerateSuiteDataDDT(unittest.TestCase): + """Suite-owned variable whose type is a DDT defined in a scheme module.""" + + def setUp(self): + self.suite_vars = { + 'volume_mixing_ratio_ddt': _make_sv( + 'volume_mixing_ratio_ddt', 'vmr', 'vmr_type', '', 'none', + dims=[], scheme='make_ddt', + ), + } + self.ddt_module_map = {'vmr_type': 'make_ddt'} + + def test_emits_use_for_ddt_module(self): + lines = _generate_suite_data( + 'ddt_suite', self.suite_vars, + ddt_module_map=self.ddt_module_map, + ) + text = '\n'.join(lines) + self.assertIn('use make_ddt, only: vmr_type', text) + # Components carry the TARGET attribute so group caps can + # pointer-assign into them (see suite_data.py docstring). + self.assertIn('type(vmr_type) :: vmr', text) + + def test_use_appears_before_implicit_none(self): + lines = _generate_suite_data( + 'ddt_suite', self.suite_vars, + ddt_module_map=self.ddt_module_map, + ) + idx_use = next(i for i, l in enumerate(lines) if 'use make_ddt' in l) + idx_impl = next(i for i, l in enumerate(lines) if 'implicit none' in l) + self.assertLess(idx_use, idx_impl) + + def test_handles_type_paren_form(self): + suite_vars = { + 'wrapped_ddt': _make_sv( + 'wrapped_ddt', 'w', 'type(my_type)', '', 'none', dims=[], + ), + } + lines = _generate_suite_data( + 'ddt_suite', suite_vars, + ddt_module_map={'my_type': 'wrap_mod'}, + ) + text = '\n'.join(lines) + self.assertIn('use wrap_mod, only: my_type', text) + + def test_missing_ddt_module_raises(self): + with self.assertRaisesRegex(CCPPError, "vmr_type"): + _generate_suite_data( + 'ddt_suite', self.suite_vars, + ddt_module_map={}, + ) + + def test_no_ddt_no_use(self): + # When suite vars are all intrinsic, no DDT USE lines are emitted. + suite_var = _make_sv('temp', 't', 'real', 'kind_phys', 'K', dims=[]) + lines = _generate_suite_data( + 'ds', {'temp': suite_var}, ddt_module_map=None, + ) + text = '\n'.join(lines) + self.assertNotIn('use make_ddt', text) + self.assertIn('use ccpp_kinds, only: kind_phys', text) + + +class TestConstituentCountDim(unittest.TestCase): + """Suite-owned var dimensioned by ``number_of_ccpp_constituents``. + + The framework owns the extent, so ``init_fields`` must allocate the field + via the per-instance constituent object's ``num_layer_vars`` member and USE + the object's module (``ccpp_host_constituents``). Regression for the + CAM-SIMA se_cslam allocate path (Fix B). + """ + + def test_dim_local_expr_resolves_to_constituent_count(self): + self.assertEqual( + _dim_local_expr('number_of_ccpp_constituents', {}, {}), + 'ccpp_model_constituents_obj(i)%num_layer_vars', + ) + + def test_collect_dim_uses_adds_constituent_module(self): + sv = {'workspace': _make_sv( + 'workspace', 'work', dims=['number_of_ccpp_constituents'])} + uses = _collect_dim_uses(sv, {}) + self.assertEqual(uses.get('ccpp_host_constituents'), + ['ccpp_model_constituents_obj']) + + def test_init_fields_allocates_with_constituent_count(self): + sv = {'workspace': _make_sv( + 'workspace', 'work', dims=['number_of_ccpp_constituents'])} + text = '\n'.join(_generate_suite_data('cdim', sv, host_dict={})) + self.assertIn( + 'use ccpp_host_constituents, only: ccpp_model_constituents_obj', + text) + self.assertIn( + 'allocate(ccpp_suite_data(i)%work(' + 'ccpp_model_constituents_obj(i)%num_layer_vars))', + text) + + +class TestWriteSuiteData(unittest.TestCase): + + def test_writes_file(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = write_suite_data('s', {}, tmpdir) + self.assertTrue(os.path.isfile(path)) + + def test_filename(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = write_suite_data('test_simple', {}, tmpdir) + self.assertEqual(os.path.basename(path), 'ccpp_test_simple_data.F90') + + def test_file_ends_with_newline(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = write_suite_data('s', {}, tmpdir) + with open(path) as fh: + self.assertTrue(fh.read().endswith('\n')) + + def test_creates_output_dir(self): + with tempfile.TemporaryDirectory() as tmpdir: + subdir = os.path.join(tmpdir, 'newdir') + write_suite_data('s', {}, subdir) + self.assertTrue(os.path.isdir(subdir)) + + def test_returns_absolute_path(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = write_suite_data('s', {}, tmpdir) + self.assertTrue(os.path.isabs(path)) + + +def load_tests(loader, tests, ignore): + import generator.suite_data as sd + tests.addTests(doctest.DocTestSuite(sd)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py new file mode 100644 index 00000000..c939fc3e --- /dev/null +++ b/unit-tests/test_suite_resolver.py @@ -0,0 +1,5246 @@ +"""Unit and integration tests for generator.suite_resolver and generator.group_cap. + +Tests are organized as: +- Unit tests for helper functions (unit conversion, subscript builder, etc.) +- Unit tests for single-argument resolution (Cases 1-4) +- Integration tests: full suite resolution using sample files + suite XML +- Group cap output tests: check generated Fortran source lines +""" + +import doctest +import os +import sys +import tempfile +import types +import unittest + +from metadata.metadata_table import _parse_lines, parse_metadata_file +from metadata.parse_tools import CCPPError, ParseContext +from metadata.variable_resolver import build_flat_host_dict, SchemeStore + +from generator.suite_resolver import ( + _normalize_unit_string, + _unit_to_id, + find_unit_conversion, + _apply_transform_formula, + _build_call_subscript, + _build_merged_subscript, + _one_dim_part, + _resolve_single_bound, + _substitute_instance_idx, + _translate_active_expr, + _root_symbol, + _local_name_conflict, + _resolve_one_arg, + _dedup_scheme_names, + resolve_suite, + iter_phase_calls, + validate_init_dimensions, + SuiteVar, + ResolvedArg, + ResolvedCall, + ResolvedGroup, + ResolvedSubcycle, + SuiteResolution, +) +from generator.group_cap import ( + _active_required_guard_lines, + _fortran_type_str, + _dim_decl, + _dim_decl_local, + _use_statements, + _generate_group_cap, + _collect_kinds_used, + _transform_comment, + write_group_cap, +) + +# --------------------------------------------------------------------------- +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_SAMPLES_DIR = os.path.join(_TESTS_DIR, 'sample_files') +_SUITE_DIR = os.path.join(_TESTS_DIR, 'sample_suite_files') + + +def _sf(name): + return os.path.join(_SAMPLES_DIR, name) + + +def _suite_file(name): + return os.path.join(_SUITE_DIR, name) + + +def _ctx(): + return ParseContext(0, 'test.meta') + + +def _parse(src, fname='t.meta'): + return _parse_lines(src.splitlines(keepends=True), fname) + + +def _load_full_host_dict(): + """Load the host_full + control_full metadata into a flat dict.""" + host_tbls = parse_metadata_file(_sf('host_full.meta')) + ctrl_tbls = parse_metadata_file(_sf('control_full.meta')) + return build_flat_host_dict(host_tbls, ctrl_tbls, []) + + +def _load_scheme_store(): + """Load temp_calc_adjust scheme.""" + tables = parse_metadata_file(_sf('scheme_multipart.meta')) + return SchemeStore.build_from(tables) + + +def _parse_suite(name='suite_test_simple.xml'): + """Parse a suite XML file and return the Suite object.""" + from generator.suite_xml import parse_suite_xml + import logging + logger = logging.getLogger('test') + with tempfile.TemporaryDirectory() as tmpdir: + return parse_suite_xml(_suite_file(name), tmpdir, logger, + skip_validation=True) + + +######################################################################## +# Tests: _normalize_unit_string +######################################################################## + +class TestNormalizeUnitString(unittest.TestCase): + + def test_bare_positive_exponent_gets_plus(self): + self.assertEqual(_normalize_unit_string('m2'), 'm+2') + self.assertEqual(_normalize_unit_string('m2 s-2'), 'm+2 s-2') + self.assertEqual(_normalize_unit_string('kg m2'), 'kg m+2') + + def test_existing_plus_unchanged(self): + self.assertEqual(_normalize_unit_string('m+2'), 'm+2') + self.assertEqual(_normalize_unit_string('m+2 s-2'), 'm+2 s-2') + + def test_negative_exponent_unchanged(self): + self.assertEqual(_normalize_unit_string('m s-1'), 'm s-1') + self.assertEqual(_normalize_unit_string('kg kg-1'), 'kg kg-1') + self.assertEqual(_normalize_unit_string('kg m-3'), 'kg m-3') + + def test_no_exponent_unchanged(self): + self.assertEqual(_normalize_unit_string('Pa'), 'Pa') + self.assertEqual(_normalize_unit_string('kg m s-2'), 'kg m s-2') + self.assertEqual(_normalize_unit_string(''), '') + + def test_idempotent(self): + once = _normalize_unit_string('m2 s-2') + twice = _normalize_unit_string(once) + self.assertEqual(once, twice) + + +######################################################################## +# Tests: _unit_to_id +######################################################################## + +class TestUnitToId(unittest.TestCase): + + def test_simple(self): + self.assertEqual(_unit_to_id('Pa'), 'Pa') + self.assertEqual(_unit_to_id('K'), 'K') + + def test_space_to_underscore(self): + self.assertEqual(_unit_to_id('m s-1'), 'm_s_minus_1') + self.assertEqual(_unit_to_id('kg kg-1'), 'kg_kg_minus_1') + + def test_positive_exponent(self): + self.assertEqual(_unit_to_id('m2 s-2'), 'm_plus_2_s_minus_2') + + def test_explicit_plus(self): + # hypothetical 'm+3' + self.assertEqual(_unit_to_id('m+3'), 'm_plus_3') + + def test_no_change(self): + self.assertEqual(_unit_to_id('radian'), 'radian') + self.assertEqual(_unit_to_id('degree'), 'degree') + + +######################################################################## +# Tests: find_unit_conversion +######################################################################## + +class TestFindUnitConversion(unittest.TestCase): + + def test_known_conversion(self): + fn = find_unit_conversion('Pa', 'hPa') + self.assertIsNotNone(fn) + formula = fn() + self.assertIn('{var}', formula) + + def test_same_unit_no_conversion(self): + self.assertIsNone(find_unit_conversion('K', 'K')) + self.assertIsNone(find_unit_conversion('Pa', 'Pa')) + + def test_equivalent_exponent_forms_no_conversion(self): + # ``m2`` and ``m+2`` are equivalent; the resolver must not treat + # them as a unit mismatch. See _normalize_unit_string. + self.assertIsNone(find_unit_conversion('m2 s-2', 'm+2 s-2')) + self.assertIsNone(find_unit_conversion('m+2 s-2', 'm2 s-2')) + self.assertIsNone(find_unit_conversion('m2', 'm+2')) + + def test_unknown_pair(self): + self.assertIsNone(find_unit_conversion('XYZ', 'ABC')) + + def test_reverse_conversion(self): + fwd = find_unit_conversion('Pa', 'hPa') + bwd = find_unit_conversion('hPa', 'Pa') + self.assertIsNotNone(fwd) + self.assertIsNotNone(bwd) + # Forward and backward are different formulae. + self.assertNotEqual(fwd(), bwd()) + + def test_m_s_conversion(self): + fn = find_unit_conversion('m s-1', 'km h-1') + self.assertIsNotNone(fn) + + +######################################################################## +# Tests: _apply_transform_formula +######################################################################## + +class TestApplyTransformFormula(unittest.TestCase): + + def test_with_kind(self): + from metadata.unit_conversion import Pa__to__hPa + result = _apply_transform_formula(Pa__to__hPa, 'pressure', 'kind_phys') + self.assertEqual(result, '1.0E-2_kind_phys*pressure') + + def test_without_kind(self): + from metadata.unit_conversion import Pa__to__hPa + result = _apply_transform_formula(Pa__to__hPa, 'pressure', '') + self.assertEqual(result, '1.0E-2*pressure') + + def test_complex_expr(self): + from metadata.unit_conversion import mm__to__m + result = _apply_transform_formula(mm__to__m, 'arr(lb:ub, 1:nlev)', 'k') + self.assertEqual(result, '1.0E-3_k*arr(lb:ub, 1:nlev)') + + +######################################################################## +# Tests: _build_call_subscript +######################################################################## + +_HOST_DICT_SRC = ''' +[ccpp-table-properties] + name = hm + type = host +[ccpp-arg-table] + name = hm + type = host +[ lb ] + standard_name = horizontal_loop_begin + units = count + dimensions = () + type = integer + protected = True +[ ub ] + standard_name = horizontal_loop_end + units = count + dimensions = () + type = integer + protected = True +[ ncols ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + units = count + dimensions = () + type = integer +[ nlevp1 ] + standard_name = vertical_interface_dimension + units = count + dimensions = () + type = integer +''' + + +class TestBuildCallSubscript(unittest.TestCase): + + def _hd(self): + tbls = _parse(_HOST_DICT_SRC) + return build_flat_host_dict(tbls, [], []) + + def test_scalar(self): + hd = self._hd() + sub, used = _build_call_subscript([], 'run', hd) + self.assertEqual(sub, '') + self.assertEqual(used, set()) + + def test_horizontal_run(self): + hd = self._hd() + sub, used = _build_call_subscript(['horizontal_dimension'], 'run', hd) + self.assertEqual(sub, '(lb:ub)') + self.assertIn('horizontal_loop_begin', used) + self.assertIn('horizontal_loop_end', used) + + def test_horizontal_init(self): + hd = self._hd() + sub, used = _build_call_subscript(['horizontal_dimension'], 'init', hd) + self.assertEqual(sub, '(lb:ub)') + + def test_vertical(self): + hd = self._hd() + sub, used = _build_call_subscript(['vertical_layer_dimension'], 'run', hd) + self.assertEqual(sub, '(1:nlev)') + self.assertIn('vertical_layer_dimension', used) + + def test_vertical_interface(self): + hd = self._hd() + sub, used = _build_call_subscript(['vertical_interface_dimension'], 'run', hd) + self.assertEqual(sub, '(1:nlevp1)') + + def test_2d_array_run(self): + hd = self._hd() + sub, used = _build_call_subscript( + ['horizontal_dimension', 'vertical_layer_dimension'], 'run', hd + ) + self.assertEqual(sub, '(lb:ub, 1:nlev)') + + def test_2d_array_init(self): + hd = self._hd() + sub, used = _build_call_subscript( + ['horizontal_dimension', 'vertical_layer_dimension'], 'init', hd + ) + self.assertEqual(sub, '(lb:ub, 1:nlev)') + + def test_arbitrary_dim(self): + src = _HOST_DICT_SRC + '''[ nspecies ] + standard_name = number_of_species + units = count + dimensions = () + type = integer +''' + hd = build_flat_host_dict(_parse(src), [], []) + sub, used = _build_call_subscript(['number_of_species'], 'run', hd) + self.assertEqual(sub, '(1:nspecies)') + self.assertIn('number_of_species', used) + + def test_unknown_dim_raises(self): + hd = self._hd() + with self.assertRaises(CCPPError) as cm: + _build_call_subscript(['unknown_dimension_xyz'], 'run', hd) + self.assertIn('unknown_dimension_xyz', str(cm.exception)) + + def test_unknown_dim_lists_available_std_names(self): + """When a dim lookup fails, the error message must enumerate + every standard name the resolver can see — sorted, with source + annotation — so the user can spot typos / case mismatches / + missing declarations at a glance.""" + hd = self._hd() + with self.assertRaises(CCPPError) as cm: + _build_call_subscript(['unknown_dimension_xyz'], 'run', hd) + msg = str(cm.exception) + self.assertIn('Available standard names', msg) + # The host_dict from _HOST_DICT_SRC contains horizontal_dimension + # at minimum — must be listed with a source tag. + self.assertIn('horizontal_dimension', msg) + self.assertIn('[host:', msg) + + def test_unknown_dim_did_you_mean_for_close_match(self): + """A near-miss spelling (case difference) surfaces a 'did you + mean' section above the full listing.""" + # Host has 'horizontal_dimension'; query with mixed case to + # trigger close-match detection. + hd = self._hd() + with self.assertRaises(CCPPError) as cm: + _build_call_subscript(['Horizontal_Dimension'], 'run', hd) + msg = str(cm.exception) + self.assertIn('Did you mean', msg) + self.assertIn('horizontal_dimension', msg) + + def test_instance_dim_without_instance_number_raises(self): + """Host metadata referencing an instance dim must declare + ``instance_number``; otherwise the resolver should error with a + clear, actionable message rather than silently mis-resolving. + """ + # Host dict lacks both instance_number AND number_of_instances. + # Asking the resolver to subscript a (number_of_instances) dim + # must raise CCPPError. + hd = self._hd() + with self.assertRaises(CCPPError) as cm: + _build_call_subscript(['number_of_instances'], 'run', hd) + msg = str(cm.exception) + self.assertIn('instance_number', msg) + self.assertIn('number_of_instances', msg) + + def test_missing_horiz_bounds_raises(self): + """Missing horizontal_loop_begin/end in host dict → CCPPError.""" + src = ''' +[ccpp-table-properties] + name = hm2 + type = host +[ccpp-arg-table] + name = hm2 + type = host +[ ncols ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +''' + hd = build_flat_host_dict(_parse(src), [], []) + with self.assertRaises(CCPPError): + _build_call_subscript(['horizontal_dimension'], 'run', hd) + + def test_horiz_range_ccpp_constant_one(self): + """ccpp_constant_one:horizontal_dimension resolves to lb:ub.""" + hd = self._hd() + sub, used = _build_call_subscript( + ['ccpp_constant_one:horizontal_dimension'], 'run', hd + ) + self.assertEqual(sub, '(lb:ub)') + self.assertIn('horizontal_loop_begin', used) + self.assertIn('horizontal_loop_end', used) + self.assertIn('horizontal_dimension', used) + + def test_horiz_range_integer_one(self): + """1:horizontal_dimension resolves to lb:ub.""" + hd = self._hd() + sub, used = _build_call_subscript( + ['1:horizontal_dimension'], 'run', hd + ) + self.assertEqual(sub, '(lb:ub)') + self.assertIn('horizontal_loop_begin', used) + self.assertIn('horizontal_loop_end', used) + + def test_horiz_range_bad_lower_raises(self): + """A lower bound other than 1/ccpp_constant_one for horizontal_dimension raises CCPPError.""" + hd = self._hd() + with self.assertRaises(CCPPError) as cm: + _build_call_subscript(['2:horizontal_dimension'], 'run', hd) + self.assertIn('horizontal_dimension', str(cm.exception)) + self.assertIn('ccpp_constant_one', str(cm.exception)) + + def test_horiz_range_named_lower_raises(self): + """A named lower bound for horizontal_dimension (not resolving to 1) raises CCPPError.""" + hd = self._hd() + with self.assertRaises(CCPPError) as cm: + _build_call_subscript(['vertical_layer_dimension:horizontal_dimension'], 'run', hd) + self.assertIn('horizontal_dimension', str(cm.exception)) + + def test_vertical_explicit_range(self): + """General lower:upper range resolved via host_dict for vertical dims.""" + src = _HOST_DICT_SRC + '''[ bot ] + standard_name = bottom_vertical_interface_index + units = count + dimensions = () + type = integer +''' + hd = build_flat_host_dict(_parse(src), [], []) + sub, used = _build_call_subscript( + ['bottom_vertical_interface_index:vertical_interface_dimension'], 'run', hd + ) + self.assertEqual(sub, '(bot:nlevp1)') + self.assertIn('bottom_vertical_interface_index', used) + self.assertIn('vertical_interface_dimension', used) + + def test_ccpp_constant_one_as_lower_vertical(self): + """ccpp_constant_one:vertical_layer_dimension is equivalent to vertical_layer_dimension.""" + hd = self._hd() + sub, _ = _build_call_subscript( + ['ccpp_constant_one:vertical_layer_dimension'], 'run', hd + ) + self.assertEqual(sub, '(1:nlev)') + + +######################################################################## +# Tests: _build_merged_subscript (sliced local_name with std-name indices) +######################################################################## + +_HOST_SLICE_SRC = _HOST_DICT_SRC + '''[ index_qv ] + standard_name = index_of_water_vapor_specific_HUMidity + units = index + dimensions = () + type = integer + protected = True +''' + + +class TestResolveSingleBoundSubstitutesScalarIdx(unittest.TestCase): + """``_resolve_single_bound`` returns the host entry's access path + for a DDT-component dim bound. The access path may carry baked-in + registered scalar-index placeholders (``(instance_number)``, + ``(thread_number)``) — those MUST be resolved to the host's local + Fortran names before the bound is spliced into a generated cap + subscript. Otherwise the cap leaks the std-name placeholder + verbatim and the Fortran compiler rejects it as "no IMPLICIT + type". Regression for the SCM phys_ps cap bug where + ``physics%Interstitial(thread_number)%nvdiff`` appeared inside the + ``vdftra`` slice expression.""" + + def _build_dict(self): + from metadata.metadata_table import _parse_lines + from metadata.variable_resolver import build_flat_host_dict + ddt_src = ( + "[ccpp-table-properties]\n name = GFS_interstitial_type\n type = ddt\n" + "[ccpp-arg-table]\n name = GFS_interstitial_type\n type = ddt\n" + "[ nvdiff ]\n standard_name = number_of_vertical_diffusion_tracers\n" + " units = count\n dimensions = ()\n type = integer\n" + "\n" + "[ccpp-table-properties]\n name = scm_phys_type\n type = ddt\n" + "[ccpp-arg-table]\n name = scm_phys_type\n type = ddt\n" + "[ Interstitial ]\n standard_name = GFS_interstitial_type_instance\n" + " units = DDT\n dimensions = (number_of_threads)\n" + " type = GFS_interstitial_type\n" + ) + host_src = ( + "[ccpp-table-properties]\n name = scm_type_defs\n type = host\n" + "[ccpp-arg-table]\n name = scm_type_defs\n type = host\n" + "[ physics ]\n standard_name = scm_physics_type_instance\n" + " units = DDT\n dimensions = ()\n type = scm_phys_type\n" + ) + ctrl_src = ( + "[ccpp-table-properties]\n name = ctrl_mod\n type = control\n" + "[ccpp-arg-table]\n name = ctrl_mod\n type = control\n" + "[ mythread ]\n standard_name = thread_number\n units = index\n" + " dimensions = ()\n type = integer\n" + ) + return build_flat_host_dict( + _parse_lines(host_src.splitlines(keepends=True), 'host.meta'), + _parse_lines(ctrl_src.splitlines(keepends=True), 'ctrl.meta'), + _parse_lines(ddt_src.splitlines(keepends=True), 'ddt.meta'), + ) + + def test_thread_number_placeholder_substituted_to_host_local(self): + hd = self._build_dict() + # Pre-condition: the baked access path carries the std-name + # placeholder (``thread_number``), not yet ``mythread``. + self.assertEqual( + hd['number_of_vertical_diffusion_tracers'].access_path, + 'physics%Interstitial(thread_number)%nvdiff', + ) + used = set() + resolved = _resolve_single_bound( + 'number_of_vertical_diffusion_tracers', hd, used, + ) + # The substitution must fire: ``thread_number`` → host local + # ``mythread``. Without it the cap leaks the std-name placeholder. + self.assertEqual( + resolved, 'physics%Interstitial(mythread)%nvdiff', + ) + self.assertNotIn('thread_number', resolved) + # The bound std name is recorded in *used* for USE-list tracking. + self.assertIn('number_of_vertical_diffusion_tracers', used) + + +class TestBuildMergedSubscript(unittest.TestCase): + """Cover the slicing-with-standard-name-index code path. + + A host local_name like ``q(:,:,index_of_water_vapor_specific_HUMidity)`` + parses into ``local_subscript=[':', ':', 'index_of_water_vapor_specific_HUMidity']``. + The mixed-case token is the CCPP standard name of the index variable; the + cap must emit the host's local name (``index_qv``) and import it. + """ + + def _hd(self): + return build_flat_host_dict(_parse(_HOST_SLICE_SRC), [], []) + + def test_resolves_mixed_case_standard_name(self): + hd = self._hd() + sub, used = _build_merged_subscript( + ['horizontal_dimension', 'vertical_layer_dimension'], + [':', ':', 'index_of_water_vapor_specific_HUMidity'], + 'run', hd, + ) + self.assertEqual(sub, '(lb:ub, 1:nlev, index_qv)') + # Lowercased standard name is reported as used so the cap emits a + # `use test_host_mod, only: index_qv` line. + self.assertIn('index_of_water_vapor_specific_humidity', used) + + def test_integer_literal_passthrough(self): + hd = self._hd() + sub, used = _build_merged_subscript( + ['horizontal_dimension', 'vertical_layer_dimension'], + [':', ':', '1'], + 'run', hd, + ) + self.assertEqual(sub, '(lb:ub, 1:nlev, 1)') + + def test_unknown_index_raises(self): + hd = self._hd() + with self.assertRaises(CCPPError): + _build_merged_subscript( + ['horizontal_dimension', 'vertical_layer_dimension'], + [':', ':', 'no_such_standard_name'], + 'run', hd, + ) + + def test_explicit_index_substitutes_scalar_idx_placeholder(self): + """Regression: an explicit subscript token like + ``index_of_water_vapor_specific_humidity`` whose ``access_path`` + carries a baked ``(instance_number)`` DDT-instance placeholder + must have that placeholder resolved to the host's local name + before being spliced into the subscript. Without the + substitution the generator emits Fortran like + ``qgrs(lb:ub, 1:nlev, GFS_Control(instance_number)%ntqv)`` and + the compiler rejects ``instance_number`` as untyped. Found + 2026-05-15 in the NEPTUNE phys_ps cap. + """ + from metadata.metadata_table import _parse_lines + ddt_src = ( + "[ccpp-table-properties]\n name = GFS_control_type\n type = ddt\n" + "[ccpp-arg-table]\n name = GFS_control_type\n type = ddt\n" + "[ ntqv ]\n" + " standard_name = index_of_water_vapor_specific_humidity\n" + " units = index\n dimensions = ()\n type = integer\n" + ) + host_src = ( + "[ccpp-table-properties]\n name = scm_type_defs\n type = host\n" + "[ccpp-arg-table]\n name = scm_type_defs\n type = host\n" + "[ GFS_Control ]\n standard_name = GFS_control_type_instance\n" + " units = DDT\n dimensions = (number_of_instances)\n" + " type = GFS_control_type\n" + "[ ncols ]\n standard_name = horizontal_dimension\n" + " units = count\n dimensions = ()\n type = integer\n" + "[ nlev ]\n standard_name = vertical_layer_dimension\n" + " units = count\n dimensions = ()\n type = integer\n" + ) + ctrl_src = ( + "[ccpp-table-properties]\n name = ctrl_mod\n type = control\n" + "[ccpp-arg-table]\n name = ctrl_mod\n type = control\n" + "[ lb ]\n standard_name = horizontal_loop_begin\n units = index\n" + " dimensions = ()\n type = integer\n" + "[ ub ]\n standard_name = horizontal_loop_end\n units = index\n" + " dimensions = ()\n type = integer\n" + "[ instance ]\n standard_name = instance_number\n units = index\n" + " dimensions = ()\n type = integer\n" + "[ ninstances ]\n standard_name = number_of_instances\n" + " units = count\n dimensions = ()\n type = integer\n" + ) + hd = build_flat_host_dict( + _parse_lines(host_src.splitlines(keepends=True), 'host.meta'), + _parse_lines(ctrl_src.splitlines(keepends=True), 'ctrl.meta'), + _parse_lines(ddt_src.splitlines(keepends=True), 'ddt.meta'), + ) + # Pre-condition: the inner ntqv entry's access path carries the + # baked ``(instance_number)`` placeholder. + self.assertEqual( + hd['index_of_water_vapor_specific_humidity'].access_path, + 'GFS_Control(instance_number)%ntqv', + ) + sub, _used = _build_merged_subscript( + ['horizontal_dimension', 'vertical_layer_dimension'], + [':', ':', 'index_of_water_vapor_specific_humidity'], + 'run', hd, + ) + # The emitted subscript must substitute the placeholder to the + # host's local name (``instance``). + self.assertEqual( + sub, '(lb:ub, 1:nlev, GFS_Control(instance)%ntqv)', + ) + self.assertNotIn('instance_number', sub) + + def test_explicit_index_nested_ddt_two_placeholder_levels(self): + """Recursive variant: the index token's access_path crosses TWO + DDT levels, each with its own registered scalar-index dim. The + baked path contains two distinct placeholders + (``(instance_number)`` outer + ``(thread_number)`` inner) plus + a third occurrence of one of them; ``_substitute_scalar_idx`` + must rewrite all of them in a single pass.""" + from metadata.metadata_table import _parse_lines + ddt_src = ( + # Innermost DDT — defines the leaf index variable. + "[ccpp-table-properties]\n name = scratch_type\n type = ddt\n" + "[ccpp-arg-table]\n name = scratch_type\n type = ddt\n" + "[ idx_qv ]\n" + " standard_name = index_of_water_vapor_specific_humidity\n" + " units = index\n dimensions = ()\n type = integer\n" + "\n" + # Middle DDT — sliced per OpenMP thread. + "[ccpp-table-properties]\n name = GFS_interstitial_type\n" + " type = ddt\n" + "[ccpp-arg-table]\n name = GFS_interstitial_type\n" + " type = ddt\n" + "[ scratch ]\n" + " standard_name = scratch_type_instance\n units = DDT\n" + " dimensions = ()\n type = scratch_type\n" + "\n" + # Outer DDT — sliced per model instance and itself carrying + # a per-thread Interstitial sub-DDT. + "[ccpp-table-properties]\n name = GFS_phys_type\n type = ddt\n" + "[ccpp-arg-table]\n name = GFS_phys_type\n type = ddt\n" + "[ Interstitial ]\n" + " standard_name = GFS_interstitial_type_instance\n" + " units = DDT\n dimensions = (number_of_threads)\n" + " type = GFS_interstitial_type\n" + ) + host_src = ( + "[ccpp-table-properties]\n name = scm_type_defs\n type = host\n" + "[ccpp-arg-table]\n name = scm_type_defs\n type = host\n" + "[ GFS_Phys ]\n standard_name = GFS_phys_type_instance\n" + " units = DDT\n dimensions = (number_of_instances)\n" + " type = GFS_phys_type\n" + "[ ncols ]\n standard_name = horizontal_dimension\n" + " units = count\n dimensions = ()\n type = integer\n" + "[ nlev ]\n standard_name = vertical_layer_dimension\n" + " units = count\n dimensions = ()\n type = integer\n" + ) + ctrl_src = ( + "[ccpp-table-properties]\n name = ctrl_mod\n type = control\n" + "[ccpp-arg-table]\n name = ctrl_mod\n type = control\n" + "[ lb ]\n standard_name = horizontal_loop_begin\n units = index\n" + " dimensions = ()\n type = integer\n" + "[ ub ]\n standard_name = horizontal_loop_end\n units = index\n" + " dimensions = ()\n type = integer\n" + "[ mythread ]\n standard_name = thread_number\n units = index\n" + " dimensions = ()\n type = integer\n" + "[ nthreads ]\n standard_name = number_of_threads\n" + " units = count\n dimensions = ()\n type = integer\n" + "[ instance ]\n standard_name = instance_number\n units = index\n" + " dimensions = ()\n type = integer\n" + "[ ninstances ]\n standard_name = number_of_instances\n" + " units = count\n dimensions = ()\n type = integer\n" + ) + hd = build_flat_host_dict( + _parse_lines(host_src.splitlines(keepends=True), 'host.meta'), + _parse_lines(ctrl_src.splitlines(keepends=True), 'ctrl.meta'), + _parse_lines(ddt_src.splitlines(keepends=True), 'ddt.meta'), + ) + # Pre-condition: the leaf entry's access path carries BOTH + # registered-scalar-index placeholders, one per DDT level. + self.assertEqual( + hd['index_of_water_vapor_specific_humidity'].access_path, + 'GFS_Phys(instance_number)%Interstitial(thread_number)%scratch%idx_qv', + ) + sub, _used = _build_merged_subscript( + ['horizontal_dimension', 'vertical_layer_dimension'], + [':', ':', 'index_of_water_vapor_specific_humidity'], + 'run', hd, + ) + # All placeholders must be resolved in one pass. + self.assertEqual( + sub, + '(lb:ub, 1:nlev, ' + 'GFS_Phys(instance)%Interstitial(mythread)%scratch%idx_qv)', + ) + self.assertNotIn('instance_number', sub) + self.assertNotIn('thread_number', sub) + + def test_explicit_index_with_literal_local_subscript(self): + """Regression 2026-05-15: a subscript-token entry whose declared + local_name carries a literal subscript (e.g. ``nstf_name(1)``) + must render the full ``(1)`` form, not bare ````. + Companion to the active-expression bug for the same root cause. + """ + from metadata.metadata_table import _parse_lines + ddt_src = ( + "[ccpp-table-properties]\n name = GFS_control_type\n type = ddt\n" + "[ccpp-arg-table]\n name = GFS_control_type\n type = ddt\n" + "[ nstf_name(1) ]\n" + " standard_name = control_for_nsstm\n" + " units = flag\n dimensions = ()\n type = integer\n" + ) + host_src = ( + "[ccpp-table-properties]\n name = scm_type_defs\n type = host\n" + "[ccpp-arg-table]\n name = scm_type_defs\n type = host\n" + "[ GFS_Control ]\n standard_name = GFS_control_type_instance\n" + " units = DDT\n dimensions = (number_of_instances)\n" + " type = GFS_control_type\n" + "[ ncols ]\n standard_name = horizontal_dimension\n" + " units = count\n dimensions = ()\n type = integer\n" + ) + ctrl_src = ( + "[ccpp-table-properties]\n name = ctrl_mod\n type = control\n" + "[ccpp-arg-table]\n name = ctrl_mod\n type = control\n" + "[ lb ]\n standard_name = horizontal_loop_begin\n units = index\n" + " dimensions = ()\n type = integer\n" + "[ ub ]\n standard_name = horizontal_loop_end\n units = index\n" + " dimensions = ()\n type = integer\n" + "[ instance ]\n standard_name = instance_number\n units = index\n" + " dimensions = ()\n type = integer\n" + "[ ninstances ]\n standard_name = number_of_instances\n" + " units = count\n dimensions = ()\n type = integer\n" + ) + hd = build_flat_host_dict( + _parse_lines(host_src.splitlines(keepends=True), 'host.meta'), + _parse_lines(ctrl_src.splitlines(keepends=True), 'ctrl.meta'), + _parse_lines(ddt_src.splitlines(keepends=True), 'ddt.meta'), + ) + sub, _used = _build_merged_subscript( + ['horizontal_dimension'], + [':', 'control_for_nsstm'], + 'run', hd, + ) + self.assertEqual( + sub, '(lb:ub, GFS_Control(instance)%nstf_name(1))', + ) + + def test_multiple_explicit_index_tokens_each_with_placeholder(self): + """Two distinct scheme-arg subscript tokens, each resolving to a + DDT-walked access path with its own ``(instance_number)`` + placeholder. Both must be substituted independently.""" + from metadata.metadata_table import _parse_lines + ddt_src = ( + "[ccpp-table-properties]\n name = GFS_control_type\n type = ddt\n" + "[ccpp-arg-table]\n name = GFS_control_type\n type = ddt\n" + "[ ntqv ]\n" + " standard_name = index_of_water_vapor_specific_humidity\n" + " units = index\n dimensions = ()\n type = integer\n" + "[ ntcw ]\n" + " standard_name = index_of_cloud_liquid_water_mixing_ratio\n" + " units = index\n dimensions = ()\n type = integer\n" + ) + host_src = ( + "[ccpp-table-properties]\n name = scm_type_defs\n type = host\n" + "[ccpp-arg-table]\n name = scm_type_defs\n type = host\n" + "[ GFS_Control ]\n standard_name = GFS_control_type_instance\n" + " units = DDT\n dimensions = (number_of_instances)\n" + " type = GFS_control_type\n" + "[ ncols ]\n standard_name = horizontal_dimension\n" + " units = count\n dimensions = ()\n type = integer\n" + "[ nlev ]\n standard_name = vertical_layer_dimension\n" + " units = count\n dimensions = ()\n type = integer\n" + ) + ctrl_src = ( + "[ccpp-table-properties]\n name = ctrl_mod\n type = control\n" + "[ccpp-arg-table]\n name = ctrl_mod\n type = control\n" + "[ lb ]\n standard_name = horizontal_loop_begin\n units = index\n" + " dimensions = ()\n type = integer\n" + "[ ub ]\n standard_name = horizontal_loop_end\n units = index\n" + " dimensions = ()\n type = integer\n" + "[ instance ]\n standard_name = instance_number\n units = index\n" + " dimensions = ()\n type = integer\n" + "[ ninstances ]\n standard_name = number_of_instances\n" + " units = count\n dimensions = ()\n type = integer\n" + ) + hd = build_flat_host_dict( + _parse_lines(host_src.splitlines(keepends=True), 'host.meta'), + _parse_lines(ctrl_src.splitlines(keepends=True), 'ctrl.meta'), + _parse_lines(ddt_src.splitlines(keepends=True), 'ddt.meta'), + ) + # Both index entries should carry the (instance_number) placeholder. + self.assertEqual( + hd['index_of_water_vapor_specific_humidity'].access_path, + 'GFS_Control(instance_number)%ntqv', + ) + self.assertEqual( + hd['index_of_cloud_liquid_water_mixing_ratio'].access_path, + 'GFS_Control(instance_number)%ntcw', + ) + # A subscript with TWO explicit index tokens — both placeholders + # must be rewritten. + sub, _used = _build_merged_subscript( + ['horizontal_dimension', 'vertical_layer_dimension'], + [':', ':', 'index_of_water_vapor_specific_humidity', + 'index_of_cloud_liquid_water_mixing_ratio'], + 'run', hd, + ) + self.assertEqual( + sub, + '(lb:ub, 1:nlev, ' + 'GFS_Control(instance)%ntqv, GFS_Control(instance)%ntcw)', + ) + self.assertNotIn('instance_number', sub) + + +######################################################################## +# Tests: _translate_active_expr +######################################################################## + +class TestTranslateActiveExpr(unittest.TestCase): + + def _hd(self): + src = ''' +[ccpp-table-properties] + name = m + type = host +[ccpp-arg-table] + name = m + type = host +[ do_something ] + standard_name = flag_for_something + units = flag + dimensions = () + type = logical +''' + return build_flat_host_dict(_parse(src), [], []) + + def test_empty(self): + hd = self._hd() + self.assertEqual(_translate_active_expr('', hd), '') + + def test_simple_replacement(self): + hd = self._hd() + result = _translate_active_expr('flag_for_something', hd) + self.assertEqual(result, 'do_something') + + def test_in_expression(self): + hd = self._hd() + result = _translate_active_expr('.not. flag_for_something', hd) + self.assertEqual(result, '.not. do_something') + + def test_unknown_preserved(self): + hd = self._hd() + result = _translate_active_expr('unknown_stdname .eqv. .true.', hd) + self.assertEqual(result, 'unknown_stdname .eqv. .true.') + + def test_ddt_component_flag_uses_full_access_path(self): + """A flag declared as a DDT-component must translate to the full + access path, not the bare component name — otherwise the + generated cap references an undefined symbol. + """ + ddt_src = ''' +[ccpp-table-properties] + name = inst_type + type = ddt +[ccpp-arg-table] + name = inst_type + type = ddt +[opt_array_flag] + standard_name = flag_for_opt_array + units = 1 + dimensions = () + type = logical +''' + host_src = ''' +[ccpp-table-properties] + name = data_mod + type = host +[ccpp-arg-table] + name = data_mod + type = host +[ncols] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +[ninstances] + standard_name = number_of_instances + units = count + dimensions = () + type = integer +[instance_data] + standard_name = instance_data + units = ddt + dimensions = (number_of_instances) + type = inst_type +''' + hd = build_flat_host_dict(_parse(host_src), [], _parse(ddt_src)) + result = _translate_active_expr('(flag_for_opt_array)', hd) + # No instance_number declared in this fixture → falls back to (1). + self.assertEqual(result, '(instance_data(1)%opt_array_flag)') + + def test_literal_subscript_in_local_name_preserved(self): + """Regression 2026-05-15: a DDT-component variable declared with a + literal subscript in its local_name (e.g. ``nstf_name(1)``) must + translate to ``(1)`` in an active expression — the + ``(1)`` carries semantic information (selects element 1 of an + integer array) and must not be dropped. Found in NEPTUNE + GFS_Statein metadata where ``tref`` had + ``active = (control_for_nsstm > 0)`` and ``control_for_nsstm`` + was declared as ``local_name = nstf_name(1)`` on GFS_Control; + the cap emitted ``GFS_Control(instance)%nstf_name > 0`` (rank + mismatch) instead of ``GFS_Control(instance)%nstf_name(1) > 0``. + """ + from metadata.metadata_table import _parse_lines + ddt_src = ( + "[ccpp-table-properties]\n name = GFS_control_type\n type = ddt\n" + "[ccpp-arg-table]\n name = GFS_control_type\n type = ddt\n" + "[ nstf_name(1) ]\n" + " standard_name = control_for_nsstm\n" + " units = flag\n dimensions = ()\n type = integer\n" + ) + host_src = ( + "[ccpp-table-properties]\n name = scm_type_defs\n type = host\n" + "[ccpp-arg-table]\n name = scm_type_defs\n type = host\n" + "[ GFS_Control ]\n standard_name = GFS_control_type_instance\n" + " units = DDT\n dimensions = (number_of_instances)\n" + " type = GFS_control_type\n" + ) + ctrl_src = ( + "[ccpp-table-properties]\n name = ctrl_mod\n type = control\n" + "[ccpp-arg-table]\n name = ctrl_mod\n type = control\n" + "[ instance ]\n standard_name = instance_number\n units = index\n" + " dimensions = ()\n type = integer\n" + "[ ninstances ]\n standard_name = number_of_instances\n" + " units = count\n dimensions = ()\n type = integer\n" + ) + hd = build_flat_host_dict( + _parse_lines(host_src.splitlines(keepends=True), 'host.meta'), + _parse_lines(ctrl_src.splitlines(keepends=True), 'ctrl.meta'), + _parse_lines(ddt_src.splitlines(keepends=True), 'ddt.meta'), + ) + # Pre-conditions: access_path strips the literal subscript; + # local_subscript captures it for re-attachment at render time. + self.assertEqual( + hd['control_for_nsstm'].access_path, + 'GFS_Control(instance_number)%nstf_name', + ) + self.assertEqual(hd['control_for_nsstm'].local_subscript, ['1']) + # Active expression must emit the full ``(1)`` subscript. + result = _translate_active_expr('(control_for_nsstm > 0)', hd) + self.assertEqual(result, '(GFS_Control(instance)%nstf_name(1) > 0)') + + +######################################################################## +# Tests: _substitute_instance_idx +######################################################################## + +class TestSubstituteInstanceIdx(unittest.TestCase): + + def _hd_with_inst(self, inst_local: str = 'inst'): + src = ''' +[ccpp-table-properties] + name = ctrl + type = control +[ccpp-arg-table] + name = ctrl + type = control +[ {inst} ] + standard_name = instance_number + units = 1 + dimensions = () + type = integer +'''.format(inst=inst_local) + return build_flat_host_dict([], _parse(src), []) + + def _hd_without_inst(self): + # An empty host_dict — no instance_number, no number_of_instances. + return {} + + def test_no_template_pass_through(self): + hd = self._hd_with_inst() + self.assertEqual( + _substitute_instance_idx('ncols', hd), 'ncols', + ) + self.assertEqual( + _substitute_instance_idx('gfs%phii(lb:ub)', hd), + 'gfs%phii(lb:ub)', + ) + + def test_template_resolved_to_local_name(self): + hd = self._hd_with_inst(inst_local='inst') + self.assertEqual( + _substitute_instance_idx( + 'instance_data(instance_number)%data_array2', hd, + ), + 'instance_data(inst)%data_array2', + ) + + def test_template_uses_local_name_alias(self): + hd = self._hd_with_inst(inst_local='my_inst') + self.assertEqual( + _substitute_instance_idx( + 'instance_data(instance_number)', hd, + ), + 'instance_data(my_inst)', + ) + + def test_template_resolves_to_one_when_no_pair(self): + # Host did not declare instance_number; single-instance API, + # internal arrays sized to 1. + hd = self._hd_without_inst() + self.assertEqual( + _substitute_instance_idx( + 'instance_data(instance_number)%data_array2', hd, + ), + 'instance_data(1)%data_array2', + ) + + +######################################################################## +# Tests: _root_symbol +######################################################################## + +class TestRootSymbol(unittest.TestCase): + + def test_plain(self): + self.assertEqual(_root_symbol('ncols'), 'ncols') + + def test_component(self): + self.assertEqual(_root_symbol('gfs_statein%phii'), 'gfs_statein') + + def test_subscript(self): + self.assertEqual(_root_symbol('gfs_statein(instance_number)%phii'), 'gfs_statein') + + def test_deep(self): + self.assertEqual(_root_symbol('outer%middle%inner'), 'outer') + + +######################################################################## +# Tests: _local_name_conflict +######################################################################## + +class TestLocalNameConflict(unittest.TestCase): + + def test_no_conflict(self): + self.assertEqual(_local_name_conflict('phii_l', set()), 'phii_l') + + def test_conflict_adds_2(self): + self.assertEqual(_local_name_conflict('phii_l', {'phii_l'}), 'phii_2_l') + + def test_conflict_adds_3(self): + self.assertEqual( + _local_name_conflict('phii_l', {'phii_l', 'phii_2_l'}), 'phii_3_l' + ) + + def test_case_insensitive_conflict(self): + """Fortran identifiers are case-insensitive: ``CP_l`` must be + treated as already-used when ``cp_l`` is in the existing set + (and vice versa). Regression: a HAFS_v0_hwrf_phys_ts cap + emitted both ``cp_l`` and ``CP_l`` side-by-side because the + check was string-equal rather than case-insensitive.""" + # Lower-then-upper. + self.assertEqual( + _local_name_conflict('CP_l', {'cp_l'}), 'CP_2_l', + ) + # Upper-then-lower. + self.assertEqual( + _local_name_conflict('cp_l', {'cp_l'}), 'cp_2_l', + ) + # Mixed-case existing entry too. + self.assertEqual( + _local_name_conflict('cp_l', {'Cp_L'.lower()}), 'cp_2_l', + ) + + +######################################################################## +# Tests: _resolve_one_arg (single argument) +######################################################################## + +class TestResolveOneArg(unittest.TestCase): + + def _host_dict(self): + return _load_full_host_dict() + + def _scheme_var(self, local, std_name, intent='in', units='1', + dims='()', type_='integer', kind='', optional=False): + from metadata.metadata_table import MetaVar + ctx = _ctx() + v = MetaVar(local, ctx) + v.set_attr('standard_name', std_name, ctx) + v.set_attr('units', units, ctx) + v.set_attr('dimensions', dims, ctx) + v.set_attr('type', type_, ctx) + v.set_attr('intent', intent, ctx) + if kind: + v.set_attr('kind', kind, ctx) + if optional: + v.set_attr('optional', 'True', ctx) + return v + + def test_case1_direct_host(self): + """Case 1: scalar host variable, no transform.""" + hd = self._host_dict() + suite_var = self._scheme_var( + 'im', 'horizontal_dimension', 'in', 'count') + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'my_scheme', set()) + self.assertEqual(arg.source, 'host') + self.assertEqual(arg.transform_case, 1) + # Scalar horizontal_dimension in run phase is synthesised from + # the chunk loop bounds (ub - lb + 1) rather than the host's + # full-domain ncols — see ``_HORIZ_DIM_STD`` in suite_resolver. + self.assertEqual(arg.call_expr, '(ub - lb + 1)') + self.assertFalse(arg.needs_transform) + + def test_case1_control_var(self): + """Control variable → source='control', no USE module.""" + hd = self._host_dict() + suite_var = self._scheme_var('thread_num', 'thread_number', 'in', '1') + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'my_scheme', set()) + self.assertEqual(arg.source, 'control') + self.assertIsNone(arg.module_name) + + def test_case1_2d_array_run(self): + """2D array in run phase → subscript applied.""" + hd = self._host_dict() + suite_var = self._scheme_var('temp', 'air_temperature', 'inout', 'K', + '(horizontal_dimension, vertical_layer_dimension)', + 'real', 'kind_phys') + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'my_scheme', set()) + # access_path = 'gt0', subscript = '(lb:ub, 1:nlev)' + self.assertEqual(arg.call_expr, 'gt0(lb:ub, 1:nlev)') + self.assertEqual(arg.transform_case, 1) + + def test_case2_suite_owned(self): + """Case 2: not in host, first use intent(out) → creates SuiteVar.""" + hd = self._host_dict() + suite_var = self._scheme_var('new_var', 'brand_new_standard_name', 'out', 'K', + '()', 'real', 'kind_phys') + suite_vars: dict = {} + arg = _resolve_one_arg(suite_var, 'run', hd, suite_vars, 'my_scheme', set()) + self.assertEqual(arg.source, 'suite') + self.assertIn('brand_new_standard_name', suite_vars) + self.assertIsNotNone(arg.suite_var) + + def test_case2_suite_owned_character_assumed_length_raises(self): + """A character variable first defined as intent(out) by a scheme with + kind=len=* is rejected: the defining scheme must give a concrete + length because the framework allocates suite-owned storage for it.""" + hd = self._host_dict() + suite_var = self._scheme_var('name', 'scheme_name', 'out', 'none', + '()', 'character', 'len=*') + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(suite_var, 'run', hd, {}, 'def_scheme', set()) + msg = str(cm.exception) + self.assertIn('scheme_name', msg) + self.assertIn('len=*', msg) + self.assertIn('def_scheme', msg) + + def test_case2_suite_owned_character_concrete_then_assumed_ok(self): + """A concrete-length definer followed by a len=* consumer/writer is + accepted: the suite var inherits the defining concrete length and the + later assumed-length declaration acts as a wildcard.""" + hd = self._host_dict() + definer = self._scheme_var('name', 'scheme_name', 'out', 'none', + '()', 'character', 'len=512') + suite_vars: dict = {} + _resolve_one_arg(definer, 'run', hd, suite_vars, 'def_scheme', set()) + self.assertEqual(suite_vars['scheme_name'].kind, 'len=512') + + consumer = self._scheme_var('nm', 'scheme_name', 'out', 'none', + '()', 'character', 'len=*') + arg = _resolve_one_arg(consumer, 'run', hd, suite_vars, 'use_scheme', + set()) + self.assertEqual(arg.source, 'suite') + # Storage length stays the defining concrete length. + self.assertEqual(suite_vars['scheme_name'].kind, 'len=512') + + def test_case3_not_found_intent_in_raises(self): + """Case 3: not in host, intent(in) → CCPPError.""" + hd = self._host_dict() + suite_var = self._scheme_var('missing', 'totally_missing_stdname', 'in', 'K') + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(suite_var, 'run', hd, {}, 'bad_scheme', set()) + self.assertIn('totally_missing_stdname', str(cm.exception)) + + def test_case4_suite_data_reuse(self): + """Case 4: variable already in suite_vars (from prior scheme).""" + hd = self._host_dict() + sv_creator = self._scheme_var('new_var', 'interstitial_var', 'out', 'K', + '()', 'real', 'kind_phys') + suite_vars: dict = {} + _resolve_one_arg(sv_creator, 'run', hd, suite_vars, 'scheme_a', set()) + + sv_reader = self._scheme_var('interstitial_in', 'interstitial_var', 'in', 'K', + '()', 'real', 'kind_phys') + arg = _resolve_one_arg(sv_reader, 'run', hd, suite_vars, 'scheme_b', set()) + self.assertEqual(arg.source, 'suite') + self.assertIsNotNone(arg.suite_var) + + def test_unit_transform_detected(self): + """Units differ → transformation required.""" + hd = self._host_dict() + # air_temperature is in K, request it in Pa... but there's no K→Pa conversion. + # Use a case with a known conversion: host has 'K', scheme expects 'K' → no xform. + # Let's not test K→Pa (no conversion), test a successful mismatch. + # Instead, add a variable with Pa units to the host dict. + src = ''' +[ccpp-table-properties] + name = press_mod + type = host +[ccpp-arg-table] + name = press_mod + type = host +[ pres ] + standard_name = air_pressure + units = Pa + dimensions = () + type = real + kind = kind_phys +''' + extra_tbls = _parse(src) + extra_hd = build_flat_host_dict(extra_tbls, [], {}) + combined = {**hd, **extra_hd} + + suite_var = self._scheme_var('p_hpa', 'air_pressure', 'in', 'hPa', '()', 'real', 'kind_phys') + arg = _resolve_one_arg(suite_var, 'run', combined, {}, 'my_scheme', set()) + self.assertTrue(arg.needs_unit_transform) + self.assertEqual(arg.transform_case, 3) + self.assertIn('temp_name', arg.__dataclass_fields__) # has temp_name field + self.assertTrue(arg.temp_name) + + def test_no_transform_same_units(self): + """Identical units → no transformation.""" + hd = self._host_dict() + suite_var = self._scheme_var( + 'im', 'horizontal_dimension', 'in', 'count') + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'my_scheme', set()) + self.assertFalse(arg.needs_transform) + + def test_unknown_unit_mismatch_raises(self): + """Units differ but no conversion known → CCPPError.""" + src = ''' +[ccpp-table-properties] + name = mod + type = host +[ccpp-arg-table] + name = mod + type = host +[ val ] + standard_name = some_value + units = xyz_unit + dimensions = () + type = real + kind = kind_phys +''' + hd = build_flat_host_dict(_parse(src), [], {}) + suite_var = self._scheme_var('v', 'some_value', 'in', 'abc_unit', '()', 'real', 'kind_phys') + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(suite_var, 'run', hd, {}, 'bad_scheme', set()) + self.assertIn('xyz_unit', str(cm.exception)) + + def test_optional_sets_ptr_name(self): + """Optional argument → ptr_name set.""" + hd = self._host_dict() + suite_var = self._scheme_var( + 'im', 'horizontal_dimension', 'in', 'count', optional=True) + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'my_scheme', set()) + self.assertTrue(arg.is_optional) + self.assertTrue(arg.ptr_name) + self.assertEqual(arg.transform_case, 2) + + +######################################################################## +# Tests: host vs scheme metadata compatibility +######################################################################## + +class TestHostSchemeCompatibility(unittest.TestCase): + """Resolver-level cross-metadata checks: the scheme's metadata must + agree with the defining source (host or suite) on type, rank, and + dimension identity. Units and character kind have their own existing + tests; numeric kind is intentionally lenient (triggers a transform + copy, see [[design_numeric_kind_silent_transform]]). + """ + + _HOST_SRC = ( + '[ccpp-table-properties]\n' + ' name = host_mod\n' + ' type = host\n' + '[ccpp-arg-table]\n' + ' name = host_mod\n' + ' type = host\n' + '[ ncols ]\n' + ' standard_name = horizontal_dimension\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + '[ nlev ]\n' + ' standard_name = vertical_layer_dimension\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + '[ nap ]\n' + ' standard_name = nap_indices\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + '[ tair ]\n' + ' standard_name = air_temperature\n' + ' units = K\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' + ' type = real\n' + ' kind = kind_phys\n' + '[ scalar_flag ]\n' + ' standard_name = a_scalar_flag\n' + ' units = 1\n' + ' dimensions = ()\n' + ' type = real\n' + ' kind = kind_phys\n' + '[ ap_arr ]\n' + ' standard_name = ap_indexed_array\n' + ' units = count\n' + ' dimensions = (nap_indices)\n' + ' type = integer\n' + ) + + def _host_dict(self): + from metadata.metadata_table import _parse_lines + ctrl_tbls = parse_metadata_file(_sf('control_full.meta')) + host_tbls = _parse_lines( + self._HOST_SRC.splitlines(keepends=True), 'h.meta', + ) + ctrl_only = [t for t in ctrl_tbls if t.table_type == 'control'] + return build_flat_host_dict(host_tbls, ctrl_only, []) + + def _scheme_var(self, std_name, type_, dims, units='K', kind='kind_phys', + intent='inout'): + from metadata.metadata_table import MetaVar + ctx = _ctx() + v = MetaVar('x', ctx) + v.set_attr('standard_name', std_name, ctx) + v.set_attr('units', units, ctx) + v.set_attr('dimensions', dims, ctx) + v.set_attr('type', type_, ctx) + if kind: + v.set_attr('kind', kind, ctx) + v.set_attr('intent', intent, ctx) + return v + + def test_host_scalar_scheme_rank1_raises(self): + """Host scalar `()` with scheme `(horizontal_dimension)` is a rank + mismatch — user-reported gap that the resolver now catches.""" + hd = self._host_dict() + sv = self._scheme_var( + 'a_scalar_flag', 'real', + '(horizontal_dimension)', units='1', + ) + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + msg = str(cm.exception) + self.assertIn('rank', msg) + self.assertIn('a_scalar_flag', msg) + + def test_host_dim_mismatch_raises(self): + """Host `(nap_indices)` with scheme `(horizontal_dimension)` — + same rank, different axis — is a metadata error.""" + hd = self._host_dict() + sv = self._scheme_var( + 'ap_indexed_array', 'integer', + '(horizontal_dimension)', units='count', kind='', + ) + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + msg = str(cm.exception) + self.assertIn('dimension', msg.lower()) + self.assertIn('nap_indices', msg) + self.assertIn('horizontal_dimension', msg) + + def test_type_mismatch_raises(self): + """Host `integer` with scheme `real` is a type-identity error + even when units and rank align.""" + hd = self._host_dict() + sv = self._scheme_var( + 'horizontal_dimension', 'real', '()', units='count', kind='', + ) + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + msg = str(cm.exception) + self.assertIn('type', msg.lower()) + + def test_default_lower_bound_spellings_equivalent(self): + """Three spellings of the default lower bound are equivalent: + bare ``X``, ``1:X``, and ``ccpp_constant_one:X``. The host's + bare ``vertical_layer_dimension`` matches any of these forms + on the scheme side.""" + hd = self._host_dict() + for sdim in ( + 'vertical_layer_dimension', + '1:vertical_layer_dimension', + 'ccpp_constant_one:vertical_layer_dimension', + ): + with self.subTest(scheme_dim=sdim): + sv = self._scheme_var( + 'air_temperature', 'real', + '(horizontal_dimension, {})'.format(sdim), + ) + arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + self.assertEqual(arg.standard_name, 'air_temperature') + + def test_nondefault_integer_lower_bound_mismatch_raises(self): + """``2:nlev`` and ``1:nlev`` are NOT the same axis — different + lower bound means different sub-range, even when both are + integer literals.""" + hd = self._host_dict() + sv = self._scheme_var( + 'air_temperature', 'real', + '(horizontal_dimension, 2:vertical_layer_dimension)', + ) + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + msg = str(cm.exception) + self.assertIn('dimension', msg.lower()) + self.assertIn('vertical_layer_dimension', msg) + self.assertIn('2:', msg) + + def test_nondefault_named_lower_bound_mismatch_raises(self): + """A non-default standard-name lower bound (``foo:nlev``) is + distinct from the default ``ccpp_constant_one:nlev``.""" + hd = self._host_dict() + sv = self._scheme_var( + 'air_temperature', 'real', + '(horizontal_dimension, ' + 'some_made_up_lower_bound:vertical_layer_dimension)', + ) + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + msg = str(cm.exception) + self.assertIn('dimension', msg.lower()) + self.assertIn('vertical_layer_dimension', msg) + self.assertIn('some_made_up_lower_bound', msg) + + def test_horizontal_loop_extent_in_scheme_dims_raises(self): + """No name aliasing in the compat check: a scheme that uses the + legacy ``horizontal_loop_extent`` in a dimension list while the + host declares ``horizontal_dimension`` is a mismatch. The + legacy-compat shim is the only place such rewriting belongs.""" + hd = self._host_dict() + sv = self._scheme_var( + 'air_temperature', 'real', + '(horizontal_loop_extent, vertical_layer_dimension)', + ) + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + msg = str(cm.exception) + self.assertIn('horizontal_dimension', msg) + self.assertIn('horizontal_loop_extent', msg) + + def test_suite_var_second_reader_mismatch_raises(self): + """First scheme with intent=out fixes the SuiteVar's type/rank; + a later scheme that reads it with mismatched dims is rejected.""" + hd = self._host_dict() + # First scheme creates the suite var. + writer = self._scheme_var( + 'an_arbitrary_suite_quantity', 'real', + '(horizontal_dimension, vertical_layer_dimension)', + units='K', intent='out', + ) + suite_vars = {} + _resolve_one_arg(writer, 'run', hd, suite_vars, 'sch_a', set()) + self.assertIn('an_arbitrary_suite_quantity', suite_vars) + # Second scheme reads it — but with a wrong rank. + reader = self._scheme_var( + 'an_arbitrary_suite_quantity', 'real', + '(horizontal_dimension)', units='K', intent='in', + ) + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(reader, 'run', hd, suite_vars, 'sch_b', set()) + msg = str(cm.exception) + self.assertIn('rank', msg) + self.assertIn('suite', msg) + + +######################################################################## +# Tests: host active + scheme arg coherence +######################################################################## + +class TestActiveHostHandling(unittest.TestCase): + """When the host declares ``active = ()`` on a variable: + + * Scheme arg ``optional = True`` -> the cap emits the pointer- + association pattern (transform_case 2 or 4); the scheme observes + PRESENT()=.false. when the condition is false. + * Scheme arg non-optional -> the resolver still succeeds (the + scheme is asserting the variable is mandatory); the group cap + emits a runtime guard that raises errflg/errmsg if the condition + is false at call time. The asymmetric optional rule in the + validator covers the metadata/Fortran consistency check; this + class only exercises resolver-level behaviour.""" + + _HOST_SRC = ( + '[ccpp-table-properties]\n' + ' name = active_host\n' + ' type = host\n' + '[ccpp-arg-table]\n' + ' name = active_host\n' + ' type = host\n' + '[ ncols ]\n' + ' standard_name = horizontal_dimension\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + '[ nlev ]\n' + ' standard_name = vertical_layer_dimension\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + '[ flag_passive ]\n' + ' standard_name = flag_for_passive_check\n' + ' units = flag\n' + ' dimensions = ()\n' + ' type = logical\n' + '[ gt0 ]\n' + ' standard_name = air_temperature\n' + ' units = K\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' + ' type = real\n' + ' kind = kind_phys\n' + ' active = (flag_for_passive_check)\n' + ) + + def _host_dict(self): + from metadata.metadata_table import _parse_lines + ctrl_tbls = parse_metadata_file(_sf('control_full.meta')) + host_tbls = _parse_lines( + self._HOST_SRC.splitlines(keepends=True), 'h.meta', + ) + ctrl_only = [t for t in ctrl_tbls if t.table_type == 'control'] + return build_flat_host_dict(host_tbls, ctrl_only, []) + + def _scheme_var(self, optional): + from metadata.metadata_table import MetaVar + ctx = _ctx() + v = MetaVar('temp', ctx) + v.set_attr('standard_name', 'air_temperature', ctx) + v.set_attr('units', 'K', ctx) + v.set_attr('dimensions', + '(horizontal_dimension, vertical_layer_dimension)', ctx) + v.set_attr('type', 'real', ctx) + v.set_attr('kind', 'kind_phys', ctx) + v.set_attr('intent', 'inout', ctx) + if optional: + v.set_attr('optional', 'True', ctx) + return v + + def test_optional_scheme_arg_passes(self): + hd = self._host_dict() + arg = _resolve_one_arg(self._scheme_var(optional=True), 'run', hd, + {}, 'my_scheme', set()) + self.assertEqual(arg.active, '(flag_for_passive_check)') + self.assertTrue(arg.is_optional) + # Cap uses pointer-association: transform_case 2 (or 4 with + # transform). Both are pointer-pattern paths. + self.assertIn(arg.transform_case, (2, 4)) + + def test_required_scheme_arg_resolves_with_active(self): + """A non-optional scheme arg paired with a host ``active = (...)`` + variable resolves cleanly: the resolver passes the active + condition through (so the group cap can emit a runtime guard) + and selects a non-pointer transform_case (1 or 3).""" + hd = self._host_dict() + arg = _resolve_one_arg(self._scheme_var(optional=False), 'run', hd, + {}, 'my_scheme', set()) + self.assertEqual(arg.active, '(flag_for_passive_check)') + # active_local is the same string here since flag_for_passive_check + # is referenced via its standard name with no rename. + self.assertIn('flag_passive', arg.active_local) + self.assertFalse(arg.is_optional) + # No pointer wrapper for a required arg; case 1 (direct) or 3 (transform). + self.assertIn(arg.transform_case, (1, 3)) + self.assertEqual(arg.ptr_name, '') + + def test_required_scheme_arg_emits_runtime_guard(self): + """Group-cap emitter renders the guard block for a non-optional + arg whose host declares ``active = (...)`` — guard is emitted at + the call indent, raises errflg/errmsg with a clear message, and + does *not* wrap the arg in a pointer.""" + hd = self._host_dict() + arg = _resolve_one_arg(self._scheme_var(optional=False), 'run', hd, + {}, 'my_scheme', set()) + guard = _active_required_guard_lines( + arg, scheme_name='my_scheme', phase='run', + errflg_local='errflg', errmsg_local='errmsg', + indent=' ', + ) + self.assertTrue(guard, "expected a non-empty guard block") + body = '\n'.join(guard) + self.assertIn('if (.not. (', body) + self.assertIn('flag_passive', body) + self.assertIn("my_scheme", body) + self.assertIn("air_temperature", body) + self.assertIn('errflg = 1', body) + self.assertIn('return', body) + + def test_optional_scheme_arg_skips_runtime_guard(self): + """The runtime guard is only for non-optional args — optional + args use the pointer-association pattern and need no guard.""" + hd = self._host_dict() + arg = _resolve_one_arg(self._scheme_var(optional=True), 'run', hd, + {}, 'my_scheme', set()) + guard = _active_required_guard_lines( + arg, scheme_name='my_scheme', phase='run', + errflg_local='errflg', errmsg_local='errmsg', + indent=' ', + ) + self.assertEqual(guard, []) + + def test_guard_skipped_when_no_error_locals(self): + """If the host did not declare ccpp_error_code/ccpp_error_message, + the guard is suppressed — there is no way to report the violation.""" + hd = self._host_dict() + arg = _resolve_one_arg(self._scheme_var(optional=False), 'run', hd, + {}, 'my_scheme', set()) + guard = _active_required_guard_lines( + arg, scheme_name='my_scheme', phase='run', + errflg_local=None, errmsg_local=None, + indent=' ', + ) + self.assertEqual(guard, []) + + +######################################################################## +# Tests: vertical-flip transform (top_at_one mismatch) +######################################################################## + +class TestVerticalFlipTransform(unittest.TestCase): + """When host and scheme disagree on ``top_at_one`` for a variable that + carries a vertical dimension, the resolver emits a flipped host-side + subscript and turns on the temp/transform pipeline so the call site + copies data through a contiguous local in scheme order. + """ + + def _build_host_and_scheme( + self, + host_top_at_one: bool = False, + scheme_top_at_one: bool = False, + host_units: str = 'K', + scheme_units: str = 'K', + intent: str = 'inout', + ): + host_src = ''' +[ccpp-table-properties] + name = mod + type = host +[ccpp-arg-table] + name = mod + type = host +[ ncols ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + units = count + dimensions = () + type = integer +[ gt0 ] + standard_name = air_temperature + units = {units} + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + top_at_one = {top} +'''.format(units=host_units, top='True' if host_top_at_one else 'False') + + ctrl_src = ''' +[ccpp-table-properties] + name = ctrl + type = control +[ccpp-arg-table] + name = ctrl + type = control +[ lb ] + standard_name = horizontal_loop_begin + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + units = index + dimensions = () + type = integer +''' + hd = build_flat_host_dict(_parse(host_src), _parse(ctrl_src), []) + + from metadata.metadata_table import MetaVar + ctx = _ctx() + suite_var = MetaVar('temp', ctx) + suite_var.set_attr('standard_name', 'air_temperature', ctx) + suite_var.set_attr('units', scheme_units, ctx) + suite_var.set_attr('dimensions', + '(horizontal_dimension, vertical_layer_dimension)', ctx) + suite_var.set_attr('type', 'real', ctx) + suite_var.set_attr('kind', 'kind_phys', ctx) + suite_var.set_attr('intent', intent, ctx) + if scheme_top_at_one: + suite_var.set_attr('top_at_one', 'True', ctx) + return hd, suite_var + + def test_no_flip_when_both_false(self): + hd, suite_var = self._build_host_and_scheme(False, False) + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) + self.assertFalse(arg.needs_vert_flip) + self.assertEqual(arg.transform_case, 1) + self.assertEqual(arg.call_expr, 'gt0(lb:ub, 1:nlev)') + + def test_no_flip_when_both_true(self): + hd, suite_var = self._build_host_and_scheme(True, True) + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) + self.assertFalse(arg.needs_vert_flip) + self.assertEqual(arg.transform_case, 1) + self.assertEqual(arg.call_expr, 'gt0(lb:ub, 1:nlev)') + + def test_flip_when_host_false_scheme_true(self): + hd, suite_var = self._build_host_and_scheme(False, True) + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) + self.assertTrue(arg.needs_vert_flip) + self.assertTrue(arg.needs_transform) + # Transform pipeline: temp local, transform_case 3, no unit conv. + self.assertEqual(arg.transform_case, 3) + self.assertTrue(arg.temp_name) + self.assertFalse(arg.needs_unit_transform) + self.assertFalse(arg.needs_kind_transform) + # Host-side subscript carries reverse stride at the vdim position. + self.assertEqual(arg.call_expr, 'gt0(lb:ub, nlev:1:-1)') + # Forward (pre-call) copies host → temp; backward (post-call) + # copies temp → host using the same flipped LHS. + self.assertEqual(arg.unit_forward, 'gt0(lb:ub, nlev:1:-1)') + self.assertEqual(arg.unit_backward, 'temp_l') + + def test_flip_when_host_true_scheme_false(self): + hd, suite_var = self._build_host_and_scheme(True, False) + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) + self.assertTrue(arg.needs_vert_flip) + self.assertEqual(arg.call_expr, 'gt0(lb:ub, nlev:1:-1)') + + def test_flip_composes_with_unit_conversion(self): + """Mismatched top_at_one AND a unit conversion → the unit-forward + formula is applied to the flipped call_expr; the temp pattern is + a single combined assignment.""" + hd, suite_var = self._build_host_and_scheme(False, True, + host_units='Pa', scheme_units='hPa') + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) + self.assertTrue(arg.needs_vert_flip) + self.assertTrue(arg.needs_unit_transform) + self.assertEqual(arg.transform_case, 3) + # The flipped subscript is embedded inside the unit-conversion expr. + self.assertIn('gt0(lb:ub, nlev:1:-1)', arg.unit_forward) + # And backward uses the temp's regular order multiplied by the + # inverse factor; LHS at the post-call site uses the same flipped + # subscript. + self.assertIn('temp_l', arg.unit_backward) + self.assertEqual(arg.call_expr, 'gt0(lb:ub, nlev:1:-1)') + + def test_intent_in_only_emits_forward(self): + hd, suite_var = self._build_host_and_scheme(False, True, intent='in') + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) + self.assertTrue(arg.needs_vert_flip) + self.assertEqual(arg.unit_forward, 'gt0(lb:ub, nlev:1:-1)') + self.assertEqual(arg.unit_backward, '') + + def test_intent_out_only_emits_backward(self): + hd, suite_var = self._build_host_and_scheme(False, True, intent='out') + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) + self.assertTrue(arg.needs_vert_flip) + self.assertEqual(arg.unit_forward, '') + self.assertEqual(arg.unit_backward, 'temp_l') + + def test_no_flip_on_scalar(self): + """top_at_one on a scalar (no vertical dim) is a no-op — flip needs + a vertical-axis dimension to operate on. + """ + host_src = ''' +[ccpp-table-properties] + name = mod + type = host +[ccpp-arg-table] + name = mod + type = host +[ scalar_thing ] + standard_name = some_scalar + units = 1 + dimensions = () + type = real + kind = kind_phys + top_at_one = True +''' + hd = build_flat_host_dict(_parse(host_src), [], []) + + from metadata.metadata_table import MetaVar + ctx = _ctx() + suite_var = MetaVar('s', ctx) + suite_var.set_attr('standard_name', 'some_scalar', ctx) + suite_var.set_attr('units', '1', ctx) + suite_var.set_attr('dimensions', '()', ctx) + suite_var.set_attr('type', 'real', ctx) + suite_var.set_attr('kind', 'kind_phys', ctx) + suite_var.set_attr('intent', 'in', ctx) + # Scheme leaves top_at_one at default (False). + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) + self.assertFalse(arg.needs_vert_flip) + + +######################################################################## +# Tests: character kind (len=) validation +######################################################################## + +class TestCharacterKindResolution(unittest.TestCase): + """len=* in scheme is always compatible; mismatched specific lengths are errors.""" + + def _host_with_char(self, kind='len=512'): + """Build a host dict with a character variable of the given kind.""" + src = ''' +[ccpp-table-properties] + name = hmod + type = host +[ccpp-arg-table] + name = hmod + type = host +[ msg ] + standard_name = my_message + units = none + dimensions = () + type = character + kind = {kind} +'''.format(kind=kind) + tbls = _parse(src) + return build_flat_host_dict(tbls, [], []) + + def _scheme_var_char(self, local, std_name, kind): + from metadata.metadata_table import MetaVar + ctx = _ctx() + v = MetaVar(local, ctx) + v.set_attr('standard_name', std_name, ctx) + v.set_attr('units', 'none', ctx) + v.set_attr('dimensions', '()', ctx) + v.set_attr('type', 'character', ctx) + v.set_attr('kind', kind, ctx) + v.set_attr('intent', 'out', ctx) + return v + + def test_len_star_compatible_with_len_512(self): + """len=* in scheme is always compatible — no transform, no error.""" + hd = self._host_with_char('len=512') + suite_var = self._scheme_var_char('msg', 'my_message', 'len=*') + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) + self.assertFalse(arg.needs_kind_transform) + self.assertEqual(arg.transform_case, 1) + self.assertEqual(arg.temp_name, '') + + def test_len_match_compatible(self): + """Same specific len=N in both host and scheme — no transform.""" + hd = self._host_with_char('len=512') + suite_var = self._scheme_var_char('msg', 'my_message', 'len=512') + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) + self.assertFalse(arg.needs_kind_transform) + + def test_len_star_in_host_raises(self): + """len=* in the host is rejected at host-dict construction: a host + character variable defines storage and must have a concrete length + (assumed length is valid only for a scheme dummy argument).""" + with self.assertRaises(CCPPError) as cm: + self._host_with_char('len=*') + self.assertIn('len=*', str(cm.exception)) + + def test_mismatched_specific_lengths_raises(self): + """Specific len=128 vs len=512 is a metadata error.""" + hd = self._host_with_char('len=512') + suite_var = self._scheme_var_char('msg', 'my_message', 'len=128') + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(suite_var, 'run', hd, {}, 'bad_scheme', set()) + self.assertIn('len=512', str(cm.exception)) + self.assertIn('len=128', str(cm.exception)) + + +######################################################################## +# Tests: pure-kind transform (real-kind cast) +######################################################################## + +class TestPureKindTransform(unittest.TestCase): + """When host and scheme metadata differ in *kind* only (no unit + mismatch, no vertical flip), the resolver must emit a real/int kind + cast as ``unit_forward``. Without this the cap declares the + transformation temporary but never assigns to it -- gfortran falls + back to implicit typing at the call site and the call sees garbage + / Inf. Regression: SCM_GFS_v17_p8 / bomex started failing with + ``alon = -Infinity`` in ``setclimaer`` after the host changed + ``scm_physical_constants`` from ``kind = kind_phys`` to + ``kind = dp`` -- a pure-kind mismatch against the GFS_rrtmg_pre + scheme args, which expect ``kind_phys``.""" + + _HOST_SRC_TEMPLATE = ''' +[ccpp-table-properties] + name = phys_const + type = host +[ccpp-arg-table] + name = phys_const + type = host +[ con_pi ] + standard_name = pi + units = none + dimensions = () + type = {type} + kind = {kind} +''' + + def _hd(self, kind='dp', type_='real'): + src = self._HOST_SRC_TEMPLATE.format(kind=kind, type=type_) + return build_flat_host_dict(_parse(src), [], []) + + def _scheme_var_pi(self, intent='in', kind='kind_phys', type_='real'): + from metadata.metadata_table import MetaVar + ctx = _ctx() + v = MetaVar('con_pi', ctx) + v.set_attr('standard_name', 'pi', ctx) + v.set_attr('units', 'none', ctx) + v.set_attr('dimensions', '()', ctx) + v.set_attr('type', type_, ctx) + v.set_attr('kind', kind, ctx) + v.set_attr('intent', intent, ctx) + return v + + def test_real_kind_mismatch_emits_real_cast_forward(self): + hd = self._hd(kind='dp') + scheme = self._scheme_var_pi(intent='in', kind='kind_phys') + arg = _resolve_one_arg(scheme, 'run', hd, {}, 'rrtmg', set()) + self.assertTrue(arg.needs_kind_transform) + self.assertTrue(arg.needs_transform) + self.assertEqual(arg.transform_case, 3) + # Temp must be both NAMED and ASSIGNED (the bug was that the temp + # was named but unit_forward stayed empty, so no assignment was + # emitted by the cap). + self.assertEqual(arg.temp_name, 'con_pi_l') + self.assertEqual(arg.unit_forward, 'real(con_pi, kind=kind_phys)') + + def test_real_kind_mismatch_emits_real_cast_backward_for_inout(self): + hd = self._hd(kind='dp') + scheme = self._scheme_var_pi(intent='inout', kind='kind_phys') + arg = _resolve_one_arg(scheme, 'run', hd, {}, 'rrtmg', set()) + self.assertEqual(arg.unit_forward, 'real(con_pi, kind=kind_phys)') + self.assertEqual(arg.unit_backward, 'real(con_pi_l, kind=dp)') + + def test_integer_kind_mismatch_emits_int_cast(self): + hd = self._hd(kind='int_8', type_='integer') + scheme = self._scheme_var_pi(intent='in', kind='int_4', + type_='integer') + arg = _resolve_one_arg(scheme, 'run', hd, {}, 'rrtmg', set()) + self.assertEqual(arg.unit_forward, 'int(con_pi, kind=int_4)') + + def test_unsupported_type_for_kind_cast_raises(self): + """A kind mismatch on a type without a kind-cast intrinsic (DDT, + logical) should raise a clear CCPPError pointing the user at + the metadata rather than silently emitting unassigned temps.""" + # Use a logical with two different kind names. + hd = self._hd(kind='lk1', type_='logical') + scheme = self._scheme_var_pi(intent='in', kind='lk2', + type_='logical') + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(scheme, 'run', hd, {}, 'rrtmg', set()) + msg = str(cm.exception) + self.assertIn("kind-cast intrinsic", msg) + self.assertIn("logical", msg) + + +######################################################################## +# Integration tests: resolve_suite +######################################################################## + +class TestResolveSuite(unittest.TestCase): + + def _resolve(self): + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + return resolve_suite(suite, store, hd), hd + + def test_groups_present(self): + suite_resolution, _ = self._resolve() + self.assertEqual(suite_resolution.suite_name, 'test_simple') + self.assertEqual(len(suite_resolution.groups), 1) + self.assertEqual(suite_resolution.groups[0].group_name, 'physics') + + def test_run_phase_calls(self): + suite_resolution, _ = self._resolve() + resolved_group = suite_resolution.groups[0] + self.assertIn('run', resolved_group.phase_calls) + calls = resolved_group.phase_calls['run'] + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0].scheme_name, 'temp_calc_adjust') + + def test_run_phase_args(self): + suite_resolution, _ = self._resolve() + calls = suite_resolution.groups[0].phase_calls['run'] + args = {a.scheme_local_name: a for a in calls[0].args} + # im = horizontal_dimension: scalar arg is synthesised from the loop + # bounds so the scheme sees the per-call chunk extent, not host ncols. + self.assertIn('im', args) + self.assertEqual(args['im'].call_expr, '(ub - lb + 1)') + # temp = air_temperature (2D, run phase subscript) + self.assertIn('temp', args) + self.assertEqual(args['temp'].call_expr, 'gt0(lb:ub, 1:nlev)') + # errmsg and errflg + self.assertIn('errmsg', args) + self.assertIn('errflg', args) + + def test_init_phase_calls(self): + suite_resolution, _ = self._resolve() + resolved_group = suite_resolution.groups[0] + self.assertIn('init', resolved_group.phase_calls) + calls = resolved_group.phase_calls['init'] + self.assertEqual(calls[0].scheme_name, 'temp_calc_adjust') + + def test_init_phase_horizontal_subscript(self): + """In init phase, scalar horizontal_dimension does not produce an lb:ub slice.""" + suite_resolution, _ = self._resolve() + resolved_group = suite_resolution.groups[0] + # temp_calc_adjust_init doesn't have temp, but the general rule should hold + # for any 2D variable in a non-run phase: no lb:ub slice in init args. + # (Scalar horizontal_dimension args are synthesised as (ub - lb + 1), + # which collapses to ncols in non-run phases but never contains + # the substring 'lb:ub'.) + calls = resolved_group.phase_calls.get('init', []) + for resolved_call in calls: + for arg in resolved_call.args: + self.assertNotIn('lb:ub', arg.call_expr) + + def test_no_suite_vars(self): + """All variables in temp_calc_adjust are provided by the host.""" + suite_resolution, _ = self._resolve() + self.assertEqual(suite_resolution.suite_vars, {}) + + def test_used_modules(self): + suite_resolution, _ = self._resolve() + calls = suite_resolution.groups[0].phase_calls['run'] + mods = calls[0].used_modules + # host_phys should appear (air_temperature, horizontal_dimension, etc.) + self.assertIn('host_phys', mods) + + def test_control_args_no_module(self): + suite_resolution, _ = self._resolve() + calls = suite_resolution.groups[0].phase_calls['run'] + ctrl = [a for a in calls[0].args if a.source == 'control'] + for c in ctrl: + self.assertIsNone(c.module_name) + + +class TestResolveSuiteLoopContextVariables(unittest.TestCase): + """``ccpp_loop_counter`` and ``ccpp_loop_extent`` are loop-context + control variables scoped to the body of a ```` block. + Scheme args declaring them MUST resolve against the generated + do-loop locals when inside a subcycle, and raise a clear error + when outside. Regression for the SCM GFS_surface_loop_control + failure where the resolver bailed with the generic 'not provided + by host' message.""" + + def _build_suite_from_xml(self, xml_src: str): + import tempfile, os + from generator.suite_xml import parse_suite_xml + from metadata.parse_tools import init_log + log = init_log('test_loop_ctx') + with tempfile.TemporaryDirectory() as tdir: + path = os.path.join(tdir, 's.xml') + with open(path, 'w') as fh: + fh.write(xml_src) + return parse_suite_xml(path, output_root=tdir, logger=log) + + _LOOP_SCHEME_SRC = ''' +[ccpp-table-properties] + name = loop_scheme + type = scheme +[ccpp-arg-table] + name = loop_scheme_run + type = scheme +[ iter ] + standard_name = ccpp_loop_counter + units = index + dimensions = () + type = integer + intent = in +[ niter ] + standard_name = ccpp_loop_extent + units = index + dimensions = () + type = integer + intent = in +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + + def _store(self): + from metadata.variable_resolver import SchemeStore + return SchemeStore.build_from( + _parse(self._LOOP_SCHEME_SRC, 'loop_scheme.meta') + ) + + def _args_by_name(self, suite_resolution): + calls = list(iter_phase_calls(suite_resolution.groups[0].phase_calls['run'])) + return {a.scheme_local_name: a for a in calls[0].args} + + def test_counter_inside_subcycle_resolves_to_loop_local(self): + xml = ( + "\n" + "\n" + " \n" + " \n" + " loop_scheme\n" + " \n" + " \n" + "\n" + ) + suite_resolution = resolve_suite(self._build_suite_from_xml(xml), + self._store(), _load_full_host_dict()) + args = self._args_by_name(suite_resolution) + self.assertEqual(args['iter'].standard_name, 'ccpp_loop_counter') + self.assertEqual(args['iter'].call_expr, 'ccpp_loop_counter') + self.assertEqual(args['iter'].source, 'control') + + def test_extent_integer_literal(self): + xml = ( + "\n" + "\n" + " \n" + " \n" + " loop_scheme\n" + " \n" + " \n" + "\n" + ) + suite_resolution = resolve_suite(self._build_suite_from_xml(xml), + self._store(), _load_full_host_dict()) + args = self._args_by_name(suite_resolution) + # ``loop=3`` is a literal — extent resolves to the same literal. + self.assertEqual(args['niter'].call_expr, '3') + + def test_extent_std_name_resolves_to_host_local(self): + """``loop=`` (e.g. host control var ``num_subcycles_for_test`` + with local name ``n_sub``) must resolve ccpp_loop_extent to the + host's local Fortran name.""" + from metadata.metadata_table import parse_metadata_file + from metadata.variable_resolver import build_flat_host_dict + host_tbls = parse_metadata_file(_sf('host_full.meta')) + ctrl_tbls = parse_metadata_file(_sf('control_full.meta')) + extra_tbls = parse_metadata_file(_sf('host_subcycle_stdname.meta')) + hd = build_flat_host_dict(host_tbls + extra_tbls, ctrl_tbls, []) + xml = ( + "\n" + "\n" + " \n" + " \n" + " loop_scheme\n" + " \n" + " \n" + "\n" + ) + suite_resolution = resolve_suite(self._build_suite_from_xml(xml), + self._store(), hd) + args = self._args_by_name(suite_resolution) + self.assertEqual(args['niter'].call_expr, 'n_sub') + + def test_outside_subcycle_raises_clear_error(self): + xml = ( + "\n" + "\n" + " \n" + " loop_scheme\n" + " \n" + "\n" + ) + with self.assertRaises(CCPPError) as ctx: + resolve_suite(self._build_suite_from_xml(xml), + self._store(), _load_full_host_dict()) + msg = str(ctx.exception) + # Names the scheme + the offending std_name + the SDF remediation. + self.assertIn('loop_scheme', msg) + self.assertIn('ccpp_loop_counter', msg) + self.assertIn('\n" + "\n" + " \n" + " temp_calc_adjust\n" + " not_a_real_scheme\n" + " also_missing\n" + " \n" + "\n" + ) + suite = self._build_suite_from_xml(xml_src) + with self.assertRaises(CCPPError) as ctx: + resolve_suite(suite, store, hd) + msg = str(ctx.exception) + # Names every missing scheme. + self.assertIn('not_a_real_scheme', msg) + self.assertIn('also_missing', msg) + # Names the suite for context. + self.assertIn('bad_suite', msg) + # Points the user at --scheme-files (or the CMake equivalent). + self.assertIn('--scheme-files', msg) + + def test_unknown_scheme_in_suite_init_raises(self): + hd = _load_full_host_dict() + store = _load_scheme_store() + xml_src = ( + "\n" + "\n" + " nowhere_to_find_me\n" + " \n" + " temp_calc_adjust\n" + " \n" + "\n" + ) + suite = self._build_suite_from_xml(xml_src) + with self.assertRaises(CCPPError) as ctx: + resolve_suite(suite, store, hd) + self.assertIn('nowhere_to_find_me', str(ctx.exception)) + + +class TestDedupSchemeNames(unittest.TestCase): + """Unit tests for the non-run-phase dedup helper.""" + + def test_no_duplicates_passthrough(self): + self.assertEqual(_dedup_scheme_names(['a', 'b', 'c']), ['a', 'b', 'c']) + + def test_consecutive_duplicate_collapsed(self): + self.assertEqual(_dedup_scheme_names(['a', 'a', 'b']), ['a', 'b']) + + def test_non_consecutive_duplicate_collapsed(self): + # First occurrence kept, later ones dropped. + self.assertEqual( + _dedup_scheme_names(['a', 'b', 'a', 'c', 'b']), + ['a', 'b', 'c'], + ) + + def test_empty_input(self): + self.assertEqual(_dedup_scheme_names([]), []) + + +class TestResolveSuiteInitFinalSchemes(unittest.TestCase): + """Suite-level ```` / ```` schemes resolve to + ``ResolvedCall`` objects attached to ``SuiteResolution.suite_init_call`` + and ``.suite_final_call`` respectively. A missing init/final phase + on the named scheme raises ``CCPPError`` at resolve time.""" + + def _build_store(self, scheme_meta_text: str): + from metadata.metadata_table import _parse_lines + from metadata.variable_resolver import SchemeStore + tbls = _parse_lines( + scheme_meta_text.splitlines(keepends=True), 'sch.meta', + ) + return SchemeStore.build_from(tbls) + + def _resolve(self, suite_xml: str, scheme_meta_text: str): + import tempfile, os, logging + from test_suite_resolver import _load_full_host_dict + from generator.suite_xml import parse_suite_xml + from generator.suite_resolver import resolve_suite + hd = _load_full_host_dict() + store = self._build_store(scheme_meta_text) + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, 's.xml') + with open(path, 'w') as fh: + fh.write(suite_xml) + suite = parse_suite_xml(path, tmp, logging.getLogger('t'), + skip_validation=True) + return resolve_suite(suite, store, hd) + + _SCHEME_META = ( + '[ccpp-table-properties]\n' + ' name = init_final_test\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = init_final_test_init\n' + ' type = scheme\n' + '[errmsg]\n' + ' standard_name = ccpp_error_message\n' + ' units = none\n' + ' dimensions = ()\n' + ' type = character\n' + ' kind = len=512\n' + ' intent = out\n' + '[errflg]\n' + ' standard_name = ccpp_error_code\n' + ' units = 1\n' + ' dimensions = ()\n' + ' type = integer\n' + ' intent = out\n' + '[ccpp-arg-table]\n' + ' name = init_final_test_final\n' + ' type = scheme\n' + '[errmsg]\n' + ' standard_name = ccpp_error_message\n' + ' units = none\n' + ' dimensions = ()\n' + ' type = character\n' + ' kind = len=512\n' + ' intent = out\n' + '[errflg]\n' + ' standard_name = ccpp_error_code\n' + ' units = 1\n' + ' dimensions = ()\n' + ' type = integer\n' + ' intent = out\n' + ) + + def test_init_call_attached(self): + suite_xml = ( + '\n' + '\n' + ' init_final_test\n' + ' \n' + '\n' + ) + suite_resolution = self._resolve(suite_xml, self._SCHEME_META) + self.assertIsNotNone(suite_resolution.suite_init_call) + self.assertEqual(suite_resolution.suite_init_call.scheme_name, 'init_final_test') + self.assertEqual(suite_resolution.suite_init_call.phase, 'init') + self.assertIsNone(suite_resolution.suite_final_call) + + def test_final_call_attached(self): + suite_xml = ( + '\n' + '\n' + ' \n' + ' init_final_test\n' + '\n' + ) + suite_resolution = self._resolve(suite_xml, self._SCHEME_META) + self.assertIsNone(suite_resolution.suite_init_call) + self.assertIsNotNone(suite_resolution.suite_final_call) + self.assertEqual(suite_resolution.suite_final_call.scheme_name, 'init_final_test') + self.assertEqual(suite_resolution.suite_final_call.phase, 'final') + + def test_both_attached(self): + suite_xml = ( + '\n' + '\n' + ' init_final_test\n' + ' \n' + ' init_final_test\n' + '\n' + ) + suite_resolution = self._resolve(suite_xml, self._SCHEME_META) + self.assertIsNotNone(suite_resolution.suite_init_call) + self.assertIsNotNone(suite_resolution.suite_final_call) + + def test_init_scheme_without_init_phase_raises(self): + """If the named scheme has no ``init`` phase in its metadata, + the resolver errors out with a clear message — silent drop + would let the SDF declaration go unused at runtime.""" + # Same scheme metadata but with init removed: only ``final``. + run_only = ( + '[ccpp-table-properties]\n' + ' name = init_final_test\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = init_final_test_run\n' + ' type = scheme\n' + '[errmsg]\n' + ' standard_name = ccpp_error_message\n' + ' units = none\n' + ' dimensions = ()\n' + ' type = character\n' + ' kind = len=512\n' + ' intent = out\n' + '[errflg]\n' + ' standard_name = ccpp_error_code\n' + ' units = 1\n' + ' dimensions = ()\n' + ' type = integer\n' + ' intent = out\n' + ) + suite_xml = ( + '\n' + '\n' + ' init_final_test\n' + ' \n' + '\n' + ) + with self.assertRaises(CCPPError) as cm: + self._resolve(suite_xml, run_only) + msg = str(cm.exception) + self.assertIn('init_final_test', msg) + self.assertIn('init', msg) + + +class TestResolveSuiteDuplicateScheme(unittest.TestCase): + """Resolve a suite where one scheme appears twice in the same group. + + Run phase must preserve both call sites (the scheme runs once per + iteration, e.g. once per constituent); non-run phases must collapse to + a single call so register/init/final entry points fire exactly once + per group. Matches the advection-test pattern where + ``apply_constituent_tendencies`` is listed twice in ``physics``. + """ + + _SUITE_XML = ( + '\n' + '\n' + ' \n' + ' temp_calc_adjust\n' + ' temp_calc_adjust\n' + ' \n' + '\n' + ) + + def _resolve(self): + hd = _load_full_host_dict() + store = _load_scheme_store() + from generator.suite_xml import parse_suite_xml + import logging + logger = logging.getLogger('test') + with tempfile.TemporaryDirectory() as tmpdir: + xml_path = os.path.join(tmpdir, 'suite_dup.xml') + with open(xml_path, 'w') as fh: + fh.write(self._SUITE_XML) + suite = parse_suite_xml(xml_path, tmpdir, logger, + skip_validation=True) + return resolve_suite(suite, store, hd) + + def test_resolve_does_not_raise(self): + # The pre-fix behaviour was a CCPPError on the second occurrence. + self.assertIsNotNone(self._resolve()) + + def test_run_phase_preserves_both_calls(self): + suite_resolution = self._resolve() + run_calls = list(iter_phase_calls(suite_resolution.groups[0].phase_calls['run'])) + self.assertEqual(len(run_calls), 2) + self.assertEqual( + [c.scheme_name for c in run_calls], + ['temp_calc_adjust', 'temp_calc_adjust'], + ) + + def test_init_phase_dedupes(self): + suite_resolution = self._resolve() + init_calls = list(iter_phase_calls(suite_resolution.groups[0].phase_calls['init'])) + self.assertEqual(len(init_calls), 1) + self.assertEqual(init_calls[0].scheme_name, 'temp_calc_adjust') + + def test_final_phase_dedupes(self): + suite_resolution = self._resolve() + final_calls = list(iter_phase_calls(suite_resolution.groups[0].phase_calls['final'])) + self.assertEqual(len(final_calls), 1) + + +######################################################################## +# Tests: group cap output +######################################################################## + +class TestDimDeclLocal(unittest.TestCase): + """``_dim_decl_local`` — local-name dim resolution with horizontal-chunk + special case. The horizontal dim must emit ``lb:ub`` (using the + host's local names for horizontal_loop_begin / horizontal_loop_end) + because the temp must match the chunk slice the scheme receives at + the call site, not the full extent.""" + + def setUp(self): + self.hd = _load_full_host_dict() + + def test_empty(self): + self.assertEqual(_dim_decl_local([], self.hd), '') + + def test_horizontal_dimension_uses_chunk_bounds(self): + # control_full.meta has horizontal_loop_begin → 'lb', + # horizontal_loop_end → 'ub'. + self.assertEqual( + _dim_decl_local(['horizontal_dimension'], self.hd), + ', dimension(lb:ub)', + ) + + def test_vertical_dim_uses_local_name(self): + # No special case for vertical dims — host's local name only. + self.assertEqual( + _dim_decl_local(['vertical_layer_dimension'], self.hd), + ', dimension(nlev)', + ) + + def test_mixed_horiz_vert(self): + self.assertEqual( + _dim_decl_local( + ['horizontal_dimension', 'vertical_layer_dimension'], self.hd, + ), + ', dimension(lb:ub, nlev)', + ) + + def test_unknown_dim_falls_back_to_std_name(self): + # No entry in host_dict → use the std name verbatim. + self.assertEqual( + _dim_decl_local(['some_unknown_dim'], self.hd), + ', dimension(some_unknown_dim)', + ) + + +class TestConstituentCountDimSubscript(unittest.TestCase): + """``number_of_ccpp_constituents`` as a *call subscript* axis. + + Any variable (host, suite-owned, or scheme) may be dimensioned by the + framework constituent count; ``_one_dim_part`` must emit a whole-axis + ``:`` for it -- not only framework-constituent args (which go through + ``_const_dim_part``). Regression for the CAM-SIMA se_cslam failure where + host vars (cflx/qbot/fracis) are dimensioned by number_of_ccpp_constituents. + """ + + def test_bare_count_is_whole_axis(self): + part, used = _one_dim_part('number_of_ccpp_constituents', 'run', {}) + self.assertEqual(part, ':') + self.assertEqual(used, set()) + + def test_explicit_lower_bound_count_is_whole_axis(self): + part, used = _one_dim_part( + 'ccpp_constant_one:number_of_ccpp_constituents', 'run', {}) + self.assertEqual(part, ':') + self.assertEqual(used, set()) + + +class TestCollectKindsUsed(unittest.TestCase): + """``_collect_kinds_used`` must collect only kind *symbols* — integer + literals and ``len=...`` specifiers are NOT module symbols and must + not enter the ``use ccpp_kinds, only: ...`` list, even though they + do flow through to the temp declarations and numeric-literal kind + suffixes (which is correct Fortran).""" + + def _fake_arg(self, kind_scheme: str = '', kind_host: str = '', + temp_name: str = 'foo_l'): + """Build a minimal ResolvedArg stand-in with the fields + ``_collect_kinds_used`` reads.""" + from unittest.mock import MagicMock + a = MagicMock() + a.temp_name = temp_name + a.kind_scheme = kind_scheme + host = MagicMock() + host.kind = kind_host + a.host_entry = host + return a + + def _fake_rg(self, args): + from unittest.mock import MagicMock + # ``iter_phase_calls`` does ``isinstance(item, ResolvedCall)`` so a + # MagicMock won't pass; build a real ResolvedCall (the only field + # ``_collect_kinds_used`` reads is ``args``). + resolved_call = ResolvedCall(scheme_name='s', phase='run', args=args) + resolved_group = MagicMock() + resolved_group.phase_calls = {'run': [resolved_call]} + return resolved_group + + def test_keeps_kind_symbols(self): + args = [self._fake_arg(kind_scheme='kind_phys'), + self._fake_arg(kind_scheme='kind_dyn')] + self.assertEqual(_collect_kinds_used(self._fake_rg(args)), + ['kind_dyn', 'kind_phys']) + + def test_drops_integer_literal_kinds(self): + """``kind = 8`` is valid Fortran but not a module symbol — must + not appear in the USE list.""" + args = [self._fake_arg(kind_scheme='8')] + self.assertEqual(_collect_kinds_used(self._fake_rg(args)), []) + + def test_drops_character_len_specifiers(self): + args = [self._fake_arg(kind_scheme='len=512')] + self.assertEqual(_collect_kinds_used(self._fake_rg(args)), []) + + def test_mixed_set(self): + args = [self._fake_arg(kind_scheme='kind_phys'), + self._fake_arg(kind_scheme='8'), + self._fake_arg(kind_scheme='len=*'), + self._fake_arg(kind_scheme='kind_phys')] + self.assertEqual(_collect_kinds_used(self._fake_rg(args)), + ['kind_phys']) + + def test_no_temp_name_skipped(self): + # Args without a transformation temp are irrelevant — no kind + # parameter is emitted into a declaration for them. + args = [self._fake_arg(kind_scheme='kind_dyn', temp_name='')] + self.assertEqual(_collect_kinds_used(self._fake_rg(args)), []) + + +class TestTransformComment(unittest.TestCase): + """The trailing inline comment must list every active transform, but + must suppress "unit conversion" when the rendered formula is the + identity (formula ``'{var}'`` for dimensionally-equivalent units). + """ + + def _arg(self, **kwargs): + from unittest.mock import MagicMock + a = MagicMock() + a.needs_unit_transform = kwargs.get('needs_unit_transform', False) + a.needs_kind_transform = kwargs.get('needs_kind_transform', False) + a.needs_vert_flip = kwargs.get('needs_vert_flip', False) + a.unit_forward = kwargs.get('unit_forward', '') + a.unit_backward = kwargs.get('unit_backward', '') + a.call_expr = kwargs.get('call_expr', '') + a.temp_name = kwargs.get('temp_name', '') + a.kind_host = kwargs.get('kind_host', '') + a.kind_scheme = kwargs.get('kind_scheme', '') + return a + + def test_no_transforms_returns_empty(self): + self.assertEqual(_transform_comment(self._arg()), '') + + def test_identity_forward_suppressed(self): + """Forward formula returns the call_expr unchanged → no comment.""" + a = self._arg( + needs_unit_transform=True, + unit_forward='gt0(lb:ub, 1:nlev)', + call_expr='gt0(lb:ub, 1:nlev)', + kind_host='kind_phys', kind_scheme='kind_phys', + ) + self.assertEqual(_transform_comment(a, reverse=False), '') + + def test_identity_backward_suppressed(self): + """Backward formula returns the temp_name unchanged → no comment.""" + a = self._arg( + needs_unit_transform=True, + unit_backward='foo_l', + temp_name='foo_l', + kind_host='kind_phys', kind_scheme='kind_phys', + ) + self.assertEqual(_transform_comment(a, reverse=True), '') + + def test_non_identity_forward_emitted(self): + """Forward formula scales the call_expr → comment lists the + unit conversion.""" + a = self._arg( + needs_unit_transform=True, + unit_forward='1.0E-3_kind_phys*gt0(lb:ub)', + call_expr='gt0(lb:ub)', + kind_host='kind_phys', kind_scheme='kind_phys', + ) + self.assertIn('unit conversion', _transform_comment(a, reverse=False)) + + def test_non_identity_backward_emitted(self): + a = self._arg( + needs_unit_transform=True, + unit_backward='1.0E+3_kind_phys*foo_l', + temp_name='foo_l', + kind_host='kind_phys', kind_scheme='kind_phys', + ) + self.assertIn('unit conversion', _transform_comment(a, reverse=True)) + + def test_vert_flip_alone_emits_flip_only(self): + """A pure vertical flip (identity unit conversion, no kind change) + still gets a comment — and it mentions only the flip, not a + spurious "unit conversion".""" + a = self._arg( + needs_vert_flip=True, + unit_forward='gt0(lb:ub, nlev:1:-1)', + call_expr='gt0(lb:ub, nlev:1:-1)', + ) + self.assertEqual(_transform_comment(a, reverse=False), + '! vertical flip (top_at_one mismatch)') + + def test_unit_and_flip_both_listed(self): + a = self._arg( + needs_unit_transform=True, + needs_vert_flip=True, + unit_forward='1.0E-3_kind_phys*gt0(lb:ub, nlev:1:-1)', + call_expr='gt0(lb:ub, nlev:1:-1)', + kind_host='kind_phys', kind_scheme='kind_phys', + ) + comment = _transform_comment(a, reverse=False) + self.assertIn('unit conversion', comment) + self.assertIn('vertical flip', comment) + + +class TestFortranTypeStr(unittest.TestCase): + + def test_real_with_kind(self): + self.assertEqual(_fortran_type_str('real', 'kind_phys'), 'real(kind=kind_phys)') + + def test_integer(self): + self.assertEqual(_fortran_type_str('integer', ''), 'integer') + + def test_character_with_len(self): + self.assertEqual(_fortran_type_str('character', 'len=512'), 'character(len=512)') + + def test_ddt_gets_type_wrap(self): + self.assertEqual(_fortran_type_str('my_ddt_type', ''), 'type(my_ddt_type)') + + def test_ddt_already_wrapped(self): + self.assertEqual(_fortran_type_str('type(my_ddt)', ''), 'type(my_ddt)') + + +class TestGenerateGroupCap(unittest.TestCase): + + def _resolve_and_generate(self): + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + suite_resolution = resolve_suite(suite, store, hd) + resolved_group = suite_resolution.groups[0] + lines = _generate_group_cap('test_simple', 'physics', resolved_group, hd) + return lines + + def test_module_header(self): + lines = self._resolve_and_generate() + text = '\n'.join(lines) + self.assertIn('module ccpp_test_simple_physics_cap', text) + self.assertIn('end module ccpp_test_simple_physics_cap', text) + + def test_header_comment(self): + lines = self._resolve_and_generate() + self.assertTrue(lines[0].startswith('!')) + + def test_use_statements_present(self): + lines = self._resolve_and_generate() + text = '\n'.join(lines) + self.assertIn('use host_phys', text) + + def test_implicit_none_private(self): + lines = self._resolve_and_generate() + text = '\n'.join(lines) + self.assertIn('implicit none', text) + self.assertIn('private', text) + + def test_public_subroutines(self): + lines = self._resolve_and_generate() + text = '\n'.join(lines) + self.assertIn('public :: physics_run', text) + + def test_contains_block(self): + lines = self._resolve_and_generate() + self.assertIn('contains', lines) + + def test_run_subroutine(self): + lines = self._resolve_and_generate() + text = '\n'.join(lines) + self.assertIn('subroutine physics_run', text) + self.assertIn('end subroutine physics_run', text) + + def test_scheme_call_present(self): + lines = self._resolve_and_generate() + text = '\n'.join(lines) + self.assertIn('call temp_calc_adjust_run', text) + + def test_keyword_args_in_call(self): + lines = self._resolve_and_generate() + text = '\n'.join(lines) + # scheme local name 'im' (horizontal_dimension) is synthesised from + # the loop-bound control vars, not taken directly from host ncols. + self.assertIn('im=(ub - lb + 1)', text) + self.assertIn('timestep=dt', text) + self.assertIn('temp=gt0(lb:ub, 1:nlev)', text) + + def test_errflg_check(self): + lines = self._resolve_and_generate() + text = '\n'.join(lines) + self.assertIn('if (errflg /= 0) return', text) + + def test_init_subroutine(self): + lines = self._resolve_and_generate() + text = '\n'.join(lines) + self.assertIn('subroutine physics_init', text) + self.assertIn('call temp_calc_adjust_init', text) + + def test_write_group_cap(self): + """write_group_cap writes the file and returns its path.""" + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + suite_resolution = resolve_suite(suite, store, hd) + resolved_group = suite_resolution.groups[0] + with tempfile.TemporaryDirectory() as tmpdir: + path = write_group_cap('test_simple', 'physics', resolved_group, hd, tmpdir) + self.assertTrue(os.path.isfile(path)) + self.assertEqual(os.path.basename(path), 'ccpp_test_simple_physics_cap.F90') + with open(path) as fh: + content = fh.read() + self.assertIn('module ccpp_test_simple_physics_cap', content) + self.assertIn('call temp_calc_adjust_run', content) + + +######################################################################## +# Tests: subcycle resolution +######################################################################## + +class TestSubcycleResolution(unittest.TestCase): + + def _resolve_subcycle(self): + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_subcycle.xml') + return resolve_suite(suite, store, hd) + + def test_run_phase_has_subcycle(self): + suite_resolution = self._resolve_subcycle() + resolved_group = suite_resolution.groups[0] + run_items = resolved_group.phase_calls['run'] + self.assertEqual(len(run_items), 1) + self.assertIsInstance(run_items[0], ResolvedSubcycle) + + def test_subcycle_loop_count(self): + suite_resolution = self._resolve_subcycle() + sub = suite_resolution.groups[0].phase_calls['run'][0] + self.assertEqual(sub.loop, '3') + + def test_subcycle_contains_scheme(self): + suite_resolution = self._resolve_subcycle() + sub = suite_resolution.groups[0].phase_calls['run'][0] + self.assertEqual(len(sub.calls), 1) + self.assertEqual(sub.calls[0].scheme_name, 'temp_calc_adjust') + + def test_init_phase_is_flat(self): + """Init phase flattens subcycles — no ResolvedSubcycle in init.""" + suite_resolution = self._resolve_subcycle() + resolved_group = suite_resolution.groups[0] + for item in resolved_group.phase_calls.get('init', []): + self.assertNotIsInstance(item, ResolvedSubcycle) + + def test_iter_phase_calls_flattens(self): + suite_resolution = self._resolve_subcycle() + resolved_group = suite_resolution.groups[0] + all_calls = list(iter_phase_calls(resolved_group.phase_calls['run'])) + self.assertEqual(len(all_calls), 1) + self.assertEqual(all_calls[0].scheme_name, 'temp_calc_adjust') + + +class TestNestedSubcycleResolution(unittest.TestCase): + """SDFs may nest ```` inside ````. The resolver + must preserve the nesting (it determines the effective iteration + count product: ``outer * inner1 * inner2 * ...``). Original capgen + semantics.""" + + def _resolve_nested(self, suite_xml: str): + import tempfile, os, logging + from test_suite_resolver import ( + _load_full_host_dict, _load_scheme_store, + ) + from generator.suite_xml import parse_suite_xml + from generator.suite_resolver import resolve_suite + hd = _load_full_host_dict() + store = _load_scheme_store() + with tempfile.TemporaryDirectory() as tmp: + xml_path = os.path.join(tmp, 's.xml') + with open(xml_path, 'w') as fh: + fh.write(suite_xml) + suite = parse_suite_xml(xml_path, tmp, logging.getLogger('t'), + skip_validation=True) + return resolve_suite(suite, store, hd) + + def test_two_deep_nesting_preserved(self): + suite_xml = ( + '\n' + '\n' + ' \n' + ' \n' + ' \n' + ' temp_calc_adjust\n' + ' \n' + ' \n' + ' \n' + '\n' + ) + suite_resolution = self._resolve_nested(suite_xml) + run = suite_resolution.groups[0].phase_calls['run'] + # Outer subcycle. + self.assertEqual(len(run), 1) + outer = run[0] + self.assertIsInstance(outer, ResolvedSubcycle) + self.assertEqual(outer.loop, '3') + # Inside the outer is one inner subcycle, NOT a flat scheme list. + self.assertEqual(len(outer.calls), 1) + inner = outer.calls[0] + self.assertIsInstance(inner, ResolvedSubcycle) + self.assertEqual(inner.loop, '2') + # And the scheme lives inside the inner subcycle. + self.assertEqual(len(inner.calls), 1) + self.assertIsInstance(inner.calls[0], ResolvedCall) + self.assertEqual(inner.calls[0].scheme_name, 'temp_calc_adjust') + + def test_three_deep_nesting_preserved(self): + suite_xml = ( + '\n' + '\n' + ' \n' + ' \n' + ' \n' + ' \n' + ' temp_calc_adjust\n' + ' \n' + ' \n' + ' \n' + ' \n' + '\n' + ) + suite_resolution = self._resolve_nested(suite_xml) + run = suite_resolution.groups[0].phase_calls['run'] + outer = run[0] + mid = outer.calls[0] + inner = mid.calls[0] + self.assertEqual([outer.loop, mid.loop, inner.loop], ['3', '2', '2']) + # Leaf is the scheme call. + self.assertEqual(len(inner.calls), 1) + self.assertIsInstance(inner.calls[0], ResolvedCall) + + def test_iter_phase_calls_recurses_through_nesting(self): + """``iter_phase_calls`` must walk through every nesting level + and yield the leaf scheme calls — used everywhere a phase needs + a flat view of its scheme calls (USE collection, etc.).""" + suite_xml = ( + '\n' + '\n' + ' \n' + ' \n' + ' \n' + ' temp_calc_adjust\n' + ' \n' + ' \n' + ' \n' + '\n' + ) + suite_resolution = self._resolve_nested(suite_xml) + run = suite_resolution.groups[0].phase_calls['run'] + calls = list(iter_phase_calls(run)) + # One scheme call, reachable through two subcycle wrappers. + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0].scheme_name, 'temp_calc_adjust') + + +######################################################################## +# Tests: _resolve_subcycle_loop_bound +######################################################################## + +class TestResolveSubcycleLoopBound(unittest.TestCase): + """Subcycle ``loop=`` attribute resolution. + + ``loop=""`` passes through verbatim; ``loop=""`` + must resolve to the host's local Fortran name (or fail with a clear + error). The returned standard name drives USE-statement / dummy-arg + threading in the group cap. + """ + + def _hd(self): + from metadata.metadata_table import _parse_lines + host_src = ''' +[ccpp-table-properties] + name = host_mod + type = host +[ccpp-arg-table] + name = host_mod + type = host +[ ncols ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +[ n_sub ] + standard_name = num_subcycles_for_effr + units = count + dimensions = () + type = integer +''' + ctrl_src = ''' +[ccpp-table-properties] + name = ctrl + type = control +[ccpp-arg-table] + name = ctrl + type = control +[ n_ctrl ] + standard_name = number_of_iterations + units = count + dimensions = () + type = integer +''' + return build_flat_host_dict( + _parse_lines(host_src.splitlines(keepends=True), 'h.meta'), + _parse_lines(ctrl_src.splitlines(keepends=True), 'c.meta'), + [], + ) + + def test_none_returns_one(self): + from generator.suite_resolver import _resolve_subcycle_loop_bound + self.assertEqual( + _resolve_subcycle_loop_bound(None, self._hd()), + ('1', ''), + ) + + def test_empty_string_returns_one(self): + from generator.suite_resolver import _resolve_subcycle_loop_bound + self.assertEqual( + _resolve_subcycle_loop_bound(' ', self._hd()), + ('1', ''), + ) + + def test_integer_literal_passes_through(self): + from generator.suite_resolver import _resolve_subcycle_loop_bound + self.assertEqual( + _resolve_subcycle_loop_bound('3', self._hd()), + ('3', ''), + ) + self.assertEqual( + _resolve_subcycle_loop_bound(' 42 ', self._hd()), + ('42', ''), + ) + + def test_std_name_resolves_to_host_local(self): + from generator.suite_resolver import _resolve_subcycle_loop_bound + local, std = _resolve_subcycle_loop_bound( + 'num_subcycles_for_effr', self._hd(), + ) + self.assertEqual(local, 'n_sub') + self.assertEqual(std, 'num_subcycles_for_effr') + + def test_std_name_case_insensitive(self): + from generator.suite_resolver import _resolve_subcycle_loop_bound + local, std = _resolve_subcycle_loop_bound( + 'Num_Subcycles_For_Effr', self._hd(), + ) + self.assertEqual(local, 'n_sub') + self.assertEqual(std, 'num_subcycles_for_effr') + + def test_std_name_resolves_to_control_local(self): + from generator.suite_resolver import _resolve_subcycle_loop_bound + local, std = _resolve_subcycle_loop_bound( + 'number_of_iterations', self._hd(), + ) + self.assertEqual(local, 'n_ctrl') + self.assertEqual(std, 'number_of_iterations') + + def test_unresolved_std_name_raises(self): + from generator.suite_resolver import _resolve_subcycle_loop_bound + with self.assertRaises(CCPPError) as cm: + _resolve_subcycle_loop_bound('totally_made_up_name', self._hd()) + msg = str(cm.exception) + self.assertIn('totally_made_up_name', msg) + self.assertIn('standard name', msg) + + def test_std_name_resolving_to_ddt_component_uses_access_path(self): + """When a CCPP standard name resolves to a DDT-component entry + (i.e. the host declares the variable inside a DDT instance), the + generated Fortran must use the *full access path* + (``phys_state%num_subcycles``), not just the bare component name + (``num_subcycles``). The bare local_name wouldn't be in scope + from inside the cap.""" + from metadata.metadata_table import _parse_lines + from generator.suite_resolver import _resolve_subcycle_loop_bound + + ddt_src = ''' +[ccpp-table-properties] + name = physics_state + type = ddt +[ccpp-arg-table] + name = physics_state + type = ddt +[num_sub] + standard_name = num_subcycles_for_effr + units = count + dimensions = () + type = integer +''' + host_src = ''' +[ccpp-table-properties] + name = test_host_mod + type = host +[ccpp-arg-table] + name = test_host_mod + type = host +[ phys_state ] + standard_name = physics_state_ddt_instance + units = ddt + dimensions = () + type = physics_state +''' + hd = build_flat_host_dict( + _parse_lines(host_src.splitlines(keepends=True), 'h.meta'), + [], + _parse_lines(ddt_src.splitlines(keepends=True), 'd.meta'), + ) + expr, std = _resolve_subcycle_loop_bound( + 'num_subcycles_for_effr', hd, + ) + self.assertEqual(expr, 'phys_state%num_sub') + self.assertEqual(std, 'num_subcycles_for_effr') + + +class TestSubcycleGroupCapOutput(unittest.TestCase): + + def setUp(self): + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_subcycle.xml') + suite_resolution = resolve_suite(suite, store, hd) + resolved_group = suite_resolution.groups[0] + self.lines = _generate_group_cap('test_subcycle', 'physics', resolved_group, hd) + self.text = '\n'.join(self.lines) + + def test_do_loop_present(self): + self.assertIn('do ccpp_loop_counter = 1, 3', self.text) + + def test_end_do_present(self): + self.assertIn('end do', self.text) + + def test_loop_counter_declared(self): + self.assertIn('integer :: ccpp_loop_counter', self.text) + + def test_scheme_call_inside_loop(self): + loop_start = self.text.index('do ccpp_loop_counter = 1, 3') + loop_end = self.text.index('end do') + loop_body = self.text[loop_start:loop_end] + self.assertIn('call temp_calc_adjust_run', loop_body) + + def test_init_not_in_do_loop(self): + """Init phase is flat — no do loop.""" + self.assertNotIn('do ccpp_loop_counter', self.text.split('subroutine physics_init')[1].split('end subroutine')[0]) + + +######################################################################## +# Tests: state machine in group cap +######################################################################## + +class TestStateMachineGroupCap(unittest.TestCase): + + def setUp(self): + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + suite_resolution = resolve_suite(suite, store, hd) + resolved_group = suite_resolution.groups[0] + self.lines = _generate_group_cap('test_simple', 'physics', resolved_group, hd) + self.text = '\n'.join(self.lines) + + def test_state_constants_declared(self): + self.assertIn('CCPP_GROUP_UNINITIALIZED = 0', self.text) + self.assertIn('CCPP_GROUP_INITIALIZED = 1', self.text) + self.assertIn('CCPP_GROUP_IN_TIMESTEP = 2', self.text) + + def test_state_array_declared(self): + self.assertIn('integer, private, allocatable :: ccpp_group_state(:)', self.text) + + def test_state_alloc_public(self): + self.assertIn('public :: physics_state_alloc', self.text) + + def test_state_dealloc_public(self): + self.assertIn('public :: physics_state_dealloc', self.text) + + def test_init_idempotent_skip(self): + # init returns silently when already INITIALIZED. + self.assertIn( + 'if (ccpp_group_state(inst_num) == CCPP_GROUP_INITIALIZED) return', + self.text, + ) + + def test_init_errors_on_invalid_state(self): + # init must error if the state is anything other than UNINITIALIZED + # or INITIALIZED (idempotent skip). + init_sub = self.text.split('subroutine physics_init')[1] + init_sub = init_sub.split('end subroutine')[0] + self.assertIn( + 'ccpp_group_state(inst_num) /= CCPP_GROUP_UNINITIALIZED', init_sub + ) + self.assertIn('errflg = 1', init_sub) + + def test_init_sets_state(self): + self.assertIn('ccpp_group_state(inst_num) = CCPP_GROUP_INITIALIZED', self.text) + + def test_final_resets_state(self): + self.assertIn('ccpp_group_state(inst_num) = CCPP_GROUP_UNINITIALIZED', self.text) + + def test_run_guards_on_in_timestep(self): + # run requires IN_TIMESTEP; otherwise sets errflg and returns. + run_sub = self.text.split('subroutine physics_run')[1] + run_sub = run_sub.split('end subroutine')[0] + self.assertIn( + 'ccpp_group_state(inst_num) /= CCPP_GROUP_IN_TIMESTEP', run_sub + ) + self.assertIn('errflg = 1', run_sub) + + def test_state_alloc_subroutine(self): + # state_alloc always takes number_of_instances as explicit arg. + self.assertIn( + 'subroutine physics_state_alloc(number_of_instances, errmsg, errflg)', + self.text, + ) + self.assertIn('allocate(ccpp_group_state(number_of_instances))', self.text) + + def test_ninstances_not_used_in_group_cap(self): + # number_of_instances is no longer USEd by the group cap module; + # it is passed as an explicit argument to state_alloc instead. + preamble = self.text.split('contains')[0] + self.assertNotIn('ninstances', preamble) + + def test_state_dealloc_subroutine(self): + self.assertIn('subroutine physics_state_dealloc(errmsg, errflg)', self.text) + self.assertIn('if (allocated(ccpp_group_state)) deallocate(ccpp_group_state)', self.text) + + def test_inst_num_in_init_args(self): + # inst_num (the local name for instance_number) must be a dummy arg of init. + init_sub = self.text.split('subroutine physics_init')[1] + init_sub = init_sub.split('end subroutine')[0] + self.assertIn('inst_num', init_sub) + + def test_inst_num_in_final_args(self): + final_sub = self.text.split('subroutine physics_final')[1] + final_sub = final_sub.split('end subroutine')[0] + self.assertIn('inst_num', final_sub) + + +######################################################################## +# Tests: suite cap calls state_alloc/dealloc +######################################################################## + +class TestSuiteCapStateCalls(unittest.TestCase): + """Suite cap calls state_alloc/dealloc — multi-instance host (host_full.meta).""" + + def setUp(self): + from generator.suite_cap import _generate_suite_cap + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + suite_resolution = resolve_suite(suite, store, hd) + # Pass host_dict so number_of_instances flows through. + lines = _generate_suite_cap('test_simple', suite_resolution, store, hd) + self.text = '\n'.join(lines) + + def test_init_calls_state_alloc_with_ninstances(self): + # host_full.meta has ninstances → number_of_instances. + self.assertIn( + 'call physics_state_alloc(ninstances, errmsg, errflg)', self.text + ) + + def test_init_subroutine_has_ninstances_arg(self): + init_sub = self.text.split('subroutine test_simple_init')[1].split('end subroutine')[0] + self.assertIn('ninstances', init_sub) + + def test_final_calls_state_dealloc(self): + self.assertIn( + 'call physics_state_dealloc(errmsg, errflg)', self.text + ) + + def test_state_alloc_imported_in_suite_cap(self): + self.assertIn('physics_state_alloc', self.text.split('contains')[0]) + + def test_state_dealloc_imported_in_suite_cap(self): + self.assertIn('physics_state_dealloc', self.text.split('contains')[0]) + + +class TestSuiteCapStateCallsSingleInstance(unittest.TestCase): + """Suite cap falls back to passing literal 1 when host has no number_of_instances.""" + + def setUp(self): + from generator.suite_cap import _generate_suite_cap + # Use full host_dict but remove number_of_instances to simulate a + # single-instance host model. + hd_full = _load_full_host_dict() + hd = {k: v for k, v in hd_full.items() if k != 'number_of_instances'} + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + suite_resolution = resolve_suite(suite, store, hd) + lines = _generate_suite_cap('test_simple', suite_resolution, store, hd) + self.text = '\n'.join(lines) + + def test_init_calls_state_alloc_with_literal_1(self): + self.assertIn( + 'call physics_state_alloc(1, errmsg, errflg)', self.text + ) + + def test_init_subroutine_has_no_ninstances_arg(self): + init_sub = self.text.split('subroutine test_simple_init')[1].split('end subroutine')[0] + self.assertNotIn('number_of_instances', init_sub) + + +######################################################################## +# Register-phase + suite-owned scalar dimension flow +######################################################################## + +def _load_register_dim_scheme_store(): + """Load the register_dim_producer + register_dim_consumer schemes.""" + tables = [] + tables.extend(parse_metadata_file(_sf('scheme_register_dim_producer.meta'))) + tables.extend(parse_metadata_file(_sf('scheme_register_dim_consumer.meta'))) + return SchemeStore.build_from(tables) + + +class TestRegisterPhaseSuiteOwnedDim(unittest.TestCase): + """A scheme writes a suite-owned scalar dimension during ``_register``; + a later scheme uses it as a dimension for an interstitial array.""" + + def setUp(self): + self.hd = _load_full_host_dict() + self.store = _load_register_dim_scheme_store() + self.suite = _parse_suite('suite_register_dim.xml') + self.suite_resolution = resolve_suite(self.suite, self.store, self.hd) + + def test_dim_inter_promoted_to_suite_var(self): + # The register-phase intent=out arg becomes a suite-owned variable. + self.assertIn( + 'dimension_for_interstitial_variable', self.suite_resolution.suite_vars, + ) + suite_var = self.suite_resolution.suite_vars['dimension_for_interstitial_variable'] + self.assertEqual(suite_var.type_, 'integer') + self.assertEqual(suite_var.dimensions, []) + self.assertEqual(suite_var.source_phase, 'register') + + def test_interstitial_var_promoted_to_suite_var(self): + # The run-phase intent=out array also becomes a suite var, dimensioned + # by the register-set scalar. + self.assertIn( + 'output_only_interstitial_variable', self.suite_resolution.suite_vars, + ) + suite_var = self.suite_resolution.suite_vars['output_only_interstitial_variable'] + self.assertEqual(suite_var.dimensions, ['dimension_for_interstitial_variable']) + + def test_register_phase_call_resolved(self): + # Group's register phase has a ResolvedCall for the producer scheme. + resolved_group = self.suite_resolution.groups[0] + register_calls = list(iter_phase_calls(resolved_group.phase_calls.get('register', []))) + self.assertEqual(len(register_calls), 1) + self.assertEqual(register_calls[0].scheme_name, 'register_dim_producer') + + def test_run_phase_dim_resolves_via_suite_var(self): + # The run-phase consumer call's interstitial_var arg's call_expr + # must reference ccpp_suite_data(...)%dim_inter as the upper bound. + resolved_group = self.suite_resolution.groups[0] + run_calls = list(iter_phase_calls(resolved_group.phase_calls.get('run', []))) + consumer = next(resolved_call for resolved_call in run_calls + if resolved_call.scheme_name == 'register_dim_consumer') + inter_arg = next(a for a in consumer.args + if a.standard_name == 'output_only_interstitial_variable') + self.assertIn('1:ccpp_suite_data', inter_arg.subscript) + self.assertIn('dim_inter', inter_arg.subscript) + + +class TestRegisterPhaseSuiteCapEmission(unittest.TestCase): + """Suite cap emits the register-phase scheme call inside _register.""" + + def setUp(self): + from generator.suite_cap import _generate_suite_cap + self.hd = _load_full_host_dict() + self.store = _load_register_dim_scheme_store() + self.suite = _parse_suite('suite_register_dim.xml') + self.suite_resolution = resolve_suite(self.suite, self.store, self.hd) + self.text = '\n'.join( + _generate_suite_cap('reg_dim', self.suite_resolution, self.store, self.hd) + ) + + def test_register_subroutine_emits_scheme_call(self): + register_body = self.text.split('subroutine reg_dim_register')[1].split( + 'end subroutine reg_dim_register' + )[0] + self.assertIn('call register_dim_producer_register', register_body) + + def test_register_subroutine_uses_scheme_module(self): + # USE statement is required so the suite cap can reference _register. + self.assertIn('use register_dim_producer', self.text) + + def test_register_subroutine_uses_suite_data(self): + # Suite-owned vars (dim_inter) are accessed through ccpp__data. + self.assertIn('use ccpp_reg_dim_data', self.text) + + def test_state_transitions_to_registered(self): + register_body = self.text.split('subroutine reg_dim_register')[1].split( + 'end subroutine reg_dim_register' + )[0] + self.assertIn( + 'ccpp_suite_state(inst_num) = CCPP_SUITE_REGISTERED', + register_body, + ) + + +######################################################################## +# Constituent registration (opt-in via type=host) +######################################################################## + +def _load_constituent_host_dict(): + """Load host_with_constituents.meta + control_full.meta + the framework + constituent DDT metadata into a host dict.""" + host_tbls = parse_metadata_file(_sf('host_with_constituents.meta')) + ctrl_tbls = parse_metadata_file(_sf('control_full.meta')) + # Pull in the framework's DDT definitions for ccpp_constituent_properties_t + # / ccpp_model_constituents_t so the host's ccpp_model_constituents_object + # resolves cleanly. These are passed as DDT tables, not host tables. + ddt_tbls = [] + fw_meta = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + 'capgen', 'src', 'ccpp_constituent_prop_mod.meta', + ) + if os.path.isfile(fw_meta): + ddt_tbls = parse_metadata_file(fw_meta) + return build_flat_host_dict(host_tbls, ctrl_tbls, ddt_tbls) + + +def _load_constituent_scheme_store(): + tables = parse_metadata_file(_sf('scheme_register_constituents.meta')) + return SchemeStore.build_from(tables) + + +class TestRegisterConstituentsResolver(unittest.TestCase): + """Resolver detects intent=out ccpp_constituent_properties_t register args + and records them in suite_res.constituent_register_calls without promoting + to suite_vars.""" + + def setUp(self): + self.hd = _load_constituent_host_dict() + self.store = _load_constituent_scheme_store() + self.suite = _parse_suite('suite_register_constituents.xml') + self.suite_resolution = resolve_suite(self.suite, self.store, self.hd) + + def test_constituent_register_calls_recorded(self): + self.assertEqual( + self.suite_resolution.constituent_register_calls, + [('register_constituents', 'dyn_const')], + ) + + def test_constituent_arg_not_promoted_to_suite_var(self): + # The constituent array is per-scheme transient — never a SuiteVar. + self.assertNotIn( + 'dynamic_constituents_for_register_test', self.suite_resolution.suite_vars, + ) + + def test_constituent_arg_marked(self): + resolved_group = self.suite_resolution.groups[0] + register_call = list(iter_phase_calls(resolved_group.phase_calls['register']))[0] + const_arg = next(a for a in register_call.args if a.is_constituent_arg) + self.assertEqual(const_arg.scheme_local_name, 'dyn_const') + self.assertEqual(const_arg.call_expr, 'scheme_consts') + + +class TestRegisterConstituentsNoHostObjectRequired(unittest.TestCase): + """Under option A the constituent object is generator-owned, so the host + is NOT required to declare ``ccpp_model_constituents_object`` — a suite + that registers constituents resolves cleanly against a regular host.""" + + def test_missing_host_object_does_not_raise(self): + hd = _load_full_host_dict() + store = _load_constituent_scheme_store() + suite = _parse_suite('suite_register_constituents.xml') + suite_resolution = resolve_suite(suite, store, hd) + # The register-phase scheme is still recorded for the suite cap to + # populate the per-suite dynamic-constituent buffer. + self.assertEqual( + suite_resolution.constituent_register_calls, + [('register_constituents', 'dyn_const')], + ) + + +class TestRegisterConstituentsSuiteCap(unittest.TestCase): + """Under option A the suite cap packs each register-phase scheme's + constituent array into the per-suite ``_dynamic_constituents`` + buffer (owned by ccpp_host_constituents). The actual merge into the + host-wide constituent object happens later in + ``ccpp_register_constituents`` — NOT in the suite cap. + """ + + def setUp(self): + from generator.suite_cap import _generate_suite_cap + self.hd = _load_constituent_host_dict() + self.store = _load_constituent_scheme_store() + self.suite = _parse_suite('suite_register_constituents.xml') + self.suite_resolution = resolve_suite(self.suite, self.store, self.hd) + self.text = '\n'.join( + _generate_suite_cap('reg_consts', self.suite_resolution, self.store, self.hd) + ) + + def test_uses_constituent_prop_type(self): + # Only the property type is USE'd; the model_constituents_t DDT is + # owned by ccpp_host_constituents. + self.assertIn( + 'use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t', + self.text, + ) + + def test_uses_per_suite_buffer(self): + # The per-suite dynamic-constituent buffer is imported from the + # host_constituents module. + self.assertIn( + 'use ccpp_host_constituents, only: reg_consts_dynamic_constituents', + self.text, + ) + + def test_no_host_object_referenced(self): + # Suite cap no longer references the host-wide constituent object + # directly; that's the host_constituents module's job. + self.assertNotIn('host_consts_obj', self.text) + self.assertNotIn('%initialize_table', self.text) + self.assertNotIn('%lock_table', self.text) + self.assertNotIn('%new_field', self.text) + + def test_register_called_once_per_scheme(self): + register_body = self.text.split('subroutine reg_consts_register')[1].split( + 'end subroutine reg_consts_register' + )[0] + # Single-pass append: each constituent scheme's _register is called + # EXACTLY ONCE (the old count+copy two-pass called it twice and broke + # non-idempotent schemes such as prescribed_aerosols_register). + self.assertEqual( + register_body.count('call register_constituents_register'), 1, + ) + self.assertNotIn('First pass', register_body) + self.assertNotIn('Second pass', register_body) + + def test_buffer_allocate(self): + # Outer wrapper-DDT array is sized to number_of_instances on first + # call; each instance starts its own slot empty (``%items(0)``) and + # appends each scheme's constituents. + self.assertIn( + 'allocate(reg_consts_dynamic_constituents(', + self.text, + ) + self.assertIn('%items(0))', self.text) + + def test_buffer_append(self): + # Each scheme's returned array is appended to the per-instance slot. + self.assertIn( + '%items = [reg_consts_dynamic_constituents(inst_num)%items, scheme_consts]', + self.text, + ) + + def test_final_tears_down_per_suite_buffer(self): + # The per-suite dynamic-constituent buffer is owned by the + # suite-cap lifecycle: filled in _register, torn down in + # _final's last-to-leave block. ccpp_deallocate_dynamic_ + # constituents must NOT touch it (would break re-register). + final_body = self.text.split('subroutine reg_consts_final')[1].split( + 'end subroutine reg_consts_final' + )[0] + self.assertIn( + 'use ccpp_host_constituents, only: reg_consts_dynamic_constituents', + final_body, + ) + self.assertIn('if (all(ccpp_suite_state == CCPP_SUITE_UNREGISTERED))', + final_body) + self.assertIn( + 'if (allocated(reg_consts_dynamic_constituents)) ' + 'deallocate(reg_consts_dynamic_constituents)', + final_body, + ) + + def test_scheme_consts_temp_declared(self): + # Local temporary in the register subroutine. + register_body = self.text.split('subroutine reg_consts_register')[1].split( + 'end subroutine reg_consts_register' + )[0] + self.assertIn( + 'type(ccpp_constituent_properties_t), allocatable :: scheme_consts(:)', + register_body, + ) + + +######################################################################## +# Provider gate: a constituent-flagged consumer that is actually +# provided by an earlier scheme's intent=out output is an interstitial, +# not a constituent (mirrors original-capgen find_variable). +######################################################################## + +_PROVIDER_GATE_SCHEMES = ''' +[ccpp-table-properties] + name = make_dry + type = scheme +[ccpp-arg-table] + name = make_dry_run + type = scheme +[ ncol ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + intent = in +[ nz ] + standard_name = vertical_layer_dimension + units = count + dimensions = () + type = integer + intent = in +[ qv_dry ] + standard_name = water_vapor_mixing_ratio_wrt_dry_air + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = out +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-table-properties] + name = use_dry + type = scheme +[ccpp-arg-table] + name = use_dry_run + type = scheme +[ ncol ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + intent = in +[ nz ] + standard_name = vertical_layer_dimension + units = count + dimensions = () + type = integer + intent = in +[ qv ] + standard_name = water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water + advected = .true. + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = in +[ qv_dry ] + standard_name = water_vapor_mixing_ratio_wrt_dry_air + advected = .true. + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + +_PROVIDER_GATE_SUITE = ( + '\n' + '\n' + ' \n' + ' make_dry\n' + ' use_dry\n' + ' \n' + '\n' +) + + +class TestConstituentProviderGate(unittest.TestCase): + """A constituent-FLAGGED consumer arg whose standard name is produced by + an earlier scheme (intent=out, unflagged) must resolve as an ordinary + suite variable -- shared with the producer -- NOT as a constituent + column, and must not be registered as a constituent. A constituent + with no provider stays a constituent. Regression for the cam-sima + kessler dry/wet mixing-ratio over-registration.""" + + @classmethod + def setUpClass(cls): + import logging + from generator.suite_xml import parse_suite_xml + hd = _load_constituent_host_dict() + store = SchemeStore.build_from(_parse(_PROVIDER_GATE_SCHEMES)) + with tempfile.TemporaryDirectory() as tmp: + sx = os.path.join(tmp, 'suite_provgate.xml') + with open(sx, 'w') as fh: + fh.write(_PROVIDER_GATE_SUITE) + suite = parse_suite_xml(sx, tmp, logging.getLogger('test'), + skip_validation=True) + cls.sr = resolve_suite(suite, store, hd) + run_calls = list(iter_phase_calls(cls.sr.groups[0].phase_calls['run'])) + cls.producer = {a.scheme_local_name: a for a in run_calls[0].args} + cls.consumer = {a.scheme_local_name: a for a in run_calls[1].args} + + def test_producer_output_is_suite_var(self): + self.assertEqual(self.producer['qv_dry'].source, 'suite') + + def test_provided_consumer_resolves_as_suite_not_constituent(self): + # qv_dry is flagged advected on the consumer, but make_dry provides + # it -> must be the shared suite var, not a constituent column. + qv_dry = self.consumer['qv_dry'] + self.assertEqual(qv_dry.source, 'suite') + self.assertNotIn('vars_layer', qv_dry.call_expr) + + def test_unprovided_constituent_stays_constituent(self): + qv = self.consumer['qv'] + self.assertEqual(qv.source, 'constituent') + self.assertIn('vars_layer', qv.call_expr) + + def test_index_names_exclude_provided_dry_var(self): + self.assertEqual( + self.sr.constituent_index_names, + ['water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water'], + ) + self.assertNotIn( + 'water_vapor_mixing_ratio_wrt_dry_air', + self.sr.constituent_index_names, + ) + + +######################################################################## +# index_of_* disambiguation: a scheme-produced index (cross-phase) is a +# suite var, NOT a constituent index (mirrors rrtmgp band indices). +######################################################################## + +_INDEX_OF_SCHEMES = ''' +[ccpp-table-properties] + name = band_setup + type = scheme +[ccpp-arg-table] + name = band_setup_init + type = scheme +[ idx_sw ] + standard_name = index_of_shortwave_band + units = index + dimensions = () + type = integer + intent = out +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-table-properties] + name = band_user + type = scheme +[ccpp-arg-table] + name = band_user_run + type = scheme +[ idx_sw ] + standard_name = index_of_shortwave_band + units = index + dimensions = () + type = integer + intent = in +[ idx_const ] + standard_name = index_of_test_constituent + units = index + dimensions = () + type = integer + intent = in +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + +# Consumer scheme deliberately listed BEFORE the producer in the SDF, to +# prove resolution is phase-driven (init before run) and NOT dependent on +# scheme text order -- exactly the rrtmgp inputs_setup / sw_cloud_optics +# arrangement. +_INDEX_OF_SUITE = ( + '\n' + '\n' + ' \n' + ' band_user\n' + ' band_setup\n' + ' \n' + '\n' +) + + +class TestIndexOfSchemeProducedVsConstituent(unittest.TestCase): + """``index_of_`` produced by a scheme (intent=out, here in the init + phase) is an ordinary suite var; a later-phase consumer (run) resolves + it from suite_vars as ``source='suite'``. A genuine ``index_of_`` + that no scheme produces stays a constituent index. Regression for the + cam-sima rrtmgp ``index_of_shortwave_band`` 'missing host variable' + failure.""" + + @classmethod + def setUpClass(cls): + import logging + from generator.suite_xml import parse_suite_xml + hd = _load_constituent_host_dict() + store = SchemeStore.build_from(_parse(_INDEX_OF_SCHEMES)) + with tempfile.TemporaryDirectory() as tmp: + sx = os.path.join(tmp, 'suite_bandtest.xml') + with open(sx, 'w') as fh: + fh.write(_INDEX_OF_SUITE) + suite = parse_suite_xml(sx, tmp, logging.getLogger('test'), + skip_validation=True) + cls.sr = resolve_suite(suite, store, hd) + init_calls = list(iter_phase_calls(cls.sr.groups[0].phase_calls['init'])) + run_calls = list(iter_phase_calls(cls.sr.groups[0].phase_calls['run'])) + cls.producer = {a.scheme_local_name: a for a in init_calls[0].args} + cls.consumer = {a.scheme_local_name: a for a in run_calls[0].args} + + def test_init_producer_is_suite_var(self): + # The init-phase intent=out index is a regular suite var, not a + # constituent index written by a scheme. + idx = self.producer['idx_sw'] + self.assertEqual(idx.source, 'suite') + + def test_run_consumer_resolves_from_suite_vars(self): + # Cross-phase: produced in init, consumed in run -> source='suite'. + idx = self.consumer['idx_sw'] + self.assertEqual(idx.source, 'suite') + self.assertNotEqual(idx.source, 'constituent') + + def test_genuine_constituent_index_unchanged(self): + # index_of_test_constituent is produced by no scheme -> still a + # constituent index. + idx = self.consumer['idx_const'] + self.assertEqual(idx.source, 'constituent') + + def test_band_index_not_registered_as_constituent(self): + self.assertNotIn('shortwave_band', ' '.join(self.sr.constituent_index_names)) + self.assertIn('test_constituent', self.sr.constituent_index_names) + + +######################################################################## +# Cross-group cross-phase provision: a variable produced by a LATER +# group's init phase must be visible to an EARLIER group's run phase, +# because at runtime all groups' init complete before any group's run. +######################################################################## + +_XGROUP_SCHEMES = ''' +[ccpp-table-properties] + name = consumer_a + type = scheme +[ccpp-arg-table] + name = consumer_a_run + type = scheme +[ val ] + standard_name = some_setup_value + units = 1 + dimensions = () + type = real | kind = kind_phys + intent = in +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-table-properties] + name = producer_b + type = scheme +[ccpp-arg-table] + name = producer_b_init + type = scheme +[ val ] + standard_name = some_setup_value + units = 1 + dimensions = () + type = real | kind = kind_phys + intent = out +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + +# The CONSUMER's group is listed FIRST and the PRODUCER's group SECOND, +# and the producer provides in the init phase while the consumer reads in +# the run phase. At runtime every group's init precedes every group's +# run, so this is valid; the resolver must reflect that (phase-major). +_XGROUP_SUITE = ( + '\n' + '\n' + ' \n' + ' consumer_a\n' + ' \n' + ' \n' + ' producer_b\n' + ' \n' + '\n' +) + + +class TestCrossGroupCrossPhaseProvision(unittest.TestCase): + """A variable produced by a later group's init phase is visible to an + earlier group's run phase (all inits precede all runs at runtime). + Resolution is phase-major, group-minor. Regression for the group-major + nesting that raised 'not provided by any prior scheme'.""" + + @classmethod + def setUpClass(cls): + import logging + from generator.suite_xml import parse_suite_xml + hd = _load_constituent_host_dict() + store = SchemeStore.build_from(_parse(_XGROUP_SCHEMES)) + with tempfile.TemporaryDirectory() as tmp: + sx = os.path.join(tmp, 'suite_xgroup.xml') + with open(sx, 'w') as fh: + fh.write(_XGROUP_SUITE) + suite = parse_suite_xml(sx, tmp, logging.getLogger('test'), + skip_validation=True) + cls.sr = resolve_suite(suite, store, hd) + + def test_consumer_resolves_producer_from_later_group(self): + grpA = self.sr.groups[0] + run_calls = list(iter_phase_calls(grpA.phase_calls['run'])) + val = {a.scheme_local_name: a for a in run_calls[0].args}['val'] + self.assertEqual(val.source, 'suite') + + def test_producer_is_suite_var(self): + grpB = self.sr.groups[1] + init_calls = list(iter_phase_calls(grpB.phase_calls['init'])) + val = {a.scheme_local_name: a for a in init_calls[0].args}['val'] + self.assertEqual(val.source, 'suite') + + +######################################################################## +# Suite-var field-name uniqueness: two distinct suite vars whose +# producing schemes share a local name must get distinct suite_data +# component names (rrtmgp lw/sw `kdist`, `hrate`). +######################################################################## + +_SHARED_LOCALNAME_SCHEMES = ''' +[ccpp-table-properties] + name = prod_lw + type = scheme +[ccpp-arg-table] + name = prod_lw_run + type = scheme +[ obj ] + standard_name = longwave_optics_object + units = none + dimensions = () + type = real | kind = kind_phys + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-table-properties] + name = prod_sw + type = scheme +[ccpp-arg-table] + name = prod_sw_run + type = scheme +[ obj ] + standard_name = shortwave_optics_object + units = none + dimensions = () + type = real | kind = kind_phys + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + +_SHARED_LOCALNAME_SUITE = ( + '\n' + '\n' + ' \n' + ' prod_lw\n' + ' prod_sw\n' + ' \n' + '\n' +) + + +class TestSuiteVarFieldNameUniqueness(unittest.TestCase): + """Two distinct suite-owned vars (different std names) first produced by + scheme args sharing a local name must get UNIQUE suite_data field names, + else the generated DDT has a duplicate component. Regression for the + rrtmgp lw/sw ``kdist`` / ``hrate`` collision.""" + + @classmethod + def setUpClass(cls): + import logging + from generator.suite_xml import parse_suite_xml + from generator.suite_data import _generate_suite_data + hd = _load_constituent_host_dict() + store = SchemeStore.build_from(_parse(_SHARED_LOCALNAME_SCHEMES)) + with tempfile.TemporaryDirectory() as tmp: + sx = os.path.join(tmp, 'suite_sharedln.xml') + with open(sx, 'w') as fh: + fh.write(_SHARED_LOCALNAME_SUITE) + suite = parse_suite_xml(sx, tmp, logging.getLogger('test'), + skip_validation=True) + cls.sr = resolve_suite(suite, store, hd) + run_calls = list(iter_phase_calls(cls.sr.groups[0].phase_calls['run'])) + cls.svars = cls.sr.suite_vars + cls.data_lines = _generate_suite_data('sharedln', cls.svars) + + def test_both_suite_vars_present(self): + self.assertIn('longwave_optics_object', self.svars) + self.assertIn('shortwave_optics_object', self.svars) + + def test_field_names_distinct(self): + lw = self.svars['longwave_optics_object'].local_name + sw = self.svars['shortwave_optics_object'].local_name + self.assertNotEqual(lw, sw) + # One keeps the bare name; the other is disambiguated. + self.assertIn('obj', (lw, sw)) + + def test_no_duplicate_component_in_generated_type(self): + decls = [l for l in self.data_lines + if 'allocatable ::' in l or l.strip().startswith('real')] + names = [l.split('::')[1].strip().split('(')[0].strip() + for l in decls if '::' in l] + self.assertEqual(len(names), len(set(names)), + 'duplicate suite_data component: {}'.format(names)) + + +######################################################################## +# Constituent auto-resolution (cam-sima-style consumer schemes) +######################################################################## + +def _load_constituent_consumer_store(): + tables = parse_metadata_file(_sf('scheme_consume_constituent.meta')) + return SchemeStore.build_from(tables) + + +class TestConstituentAutoResolution(unittest.TestCase): + """Resolver routes constituent-flagged scheme args without a host or + earlier-scheme provider through the framework's ``ccpp_constituents`` + and ``ccpp_constituent_tendencies`` arrays. The suite cap will own + those arrays; the resolver emits ``source='constituent'`` ResolvedArgs. + """ + + def setUp(self): + self.hd = _load_constituent_host_dict() + self.store = _load_constituent_consumer_store() + self.suite = _parse_suite('suite_consume_constituent.xml') + self.suite_resolution = resolve_suite(self.suite, self.store, self.hd) + run_calls = list(iter_phase_calls(self.suite_resolution.groups[0].phase_calls['run'])) + self.run_args = {a.scheme_local_name: a for a in run_calls[0].args} + + def test_uses_constituents_flag_set(self): + self.assertTrue(self.suite_resolution.uses_constituents) + + def test_constituent_index_names_enumerated(self): + # Both the base read and the tendency write reference the same + # base std name. + self.assertEqual( + self.suite_resolution.constituent_index_names, + ['cloud_liquid_water_mixing_ratio'], + ) + + def test_base_constituent_call_expr(self): + cldliq = self.run_args['cldliq'] + self.assertEqual(cldliq.source, 'constituent') + # Per-instance access: ccpp_model_constituents_obj()%vars_layer(...). + # host_with_constituents.meta declares instance_number with local + # name ``inst_num`` (via control_full.meta), and number_of_instances + # with local name ``ninstances`` — but only inst_num appears here. + self.assertEqual( + cldliq.call_expr, + 'ccpp_model_constituents_obj(inst_num)%vars_layer(lb:ub, ' + '1:nlev, index_of_cloud_liquid_water_mixing_ratio)', + ) + + def test_tendency_call_expr(self): + tend = self.run_args['tend_cldliq'] + self.assertEqual(tend.source, 'constituent') + self.assertEqual( + tend.call_expr, + 'ccpp_model_constituents_obj(inst_num)%vars_layer_tend(lb:ub, ' + '1:nlev, index_of_cloud_liquid_water_mixing_ratio)', + ) + + + def test_instance_number_in_used_dim_std_names(self): + # Drives the group cap to inject instance_number as a dummy arg. + for name in ('cldliq', 'tend_cldliq'): + self.assertIn('instance_number', + self.run_args[name].used_dim_std_names) + + def test_constituent_module_name_set(self): + cldliq = self.run_args['cldliq'] + # Under option A all constituent symbols live in a single + # host-wide module, not per-suite. + self.assertEqual(cldliq.constituent_module_name, 'ccpp_host_constituents') + + def test_index_symbol_in_extras(self): + cldliq = self.run_args['cldliq'] + self.assertIn( + 'index_of_cloud_liquid_water_mixing_ratio', + cldliq.constituent_extra_symbols, + ) + + def test_constituent_args_excluded_from_introspection(self): + # source != 'host' — constituent args do not appear in suite + # input/output lists (validated indirectly via _collect_host_io + # in host_cap tests; here we just confirm the source). + for arg in self.run_args.values(): + if arg.scheme_local_name in ('cldliq', 'tend_cldliq'): + self.assertNotEqual(arg.source, 'host') + + def test_no_number_of_ccpp_constituents_in_extras(self): + # Regression: under the per-instance design, the + # ``number_of_ccpp_constituents`` symbol is no longer module-level + # — its value is reached as ccpp_model_constituents_obj(inst)% + # num_layer_vars. Even when a scheme dim references it, the + # resolver MUST NOT add it to constituent_extra_symbols (it + # doesn't exist as a USE'd symbol in ccpp_host_constituents and + # would produce an "Symbol ... not found" Fortran error). It + # must also NOT leak into used_dim_std_names (would trigger + # spurious USE/dummy-arg plumbing in the group cap). + for arg in self.run_args.values(): + self.assertNotIn( + 'number_of_ccpp_constituents', + arg.constituent_extra_symbols, + 'arg {!r} leaked number_of_ccpp_constituents into ' + 'constituent_extra_symbols'.format(arg.scheme_local_name), + ) + self.assertNotIn( + 'number_of_ccpp_constituents', + arg.used_dim_std_names, + 'arg {!r} leaked number_of_ccpp_constituents into ' + 'used_dim_std_names (should live on ' + 'used_const_dim_std_names instead)'.format( + arg.scheme_local_name), + ) + + +class TestUsedConstDimStdNames(unittest.TestCase): + """``ResolvedArg.used_const_dim_std_names`` carries framework- + constituent dim refs (notably ``number_of_ccpp_constituents``) so + the introspection routines in :mod:`generator.host_cap` can list + them as inputs without polluting the host-side + :attr:`used_dim_std_names` channel.""" + + def _scheme_var(self, local, std_name, dims, intent='in'): + from metadata.metadata_table import MetaVar + ctx = _ctx() + v = MetaVar(local, ctx) + v.set_attr('standard_name', std_name, ctx) + v.set_attr('units', 'none', ctx) + v.set_attr('dimensions', dims, ctx) + v.set_attr('type', 'real', ctx) + v.set_attr('kind', 'kind_phys', ctx) + v.set_attr('intent', intent, ctx) + return v + + def test_ccpp_constituents_dim_lands_on_dedicated_field(self): + # Uses _load_full_host_dict (no ccpp_model_constituents_t DDT + # instance), so the host-wins gate does NOT fire and the + # resolver routes through capgen's auto-provisioning path. + from generator.suite_resolver import _resolve_constituent_arg + hd = _load_full_host_dict() + suite_var = self._scheme_var( + 'consts', 'ccpp_constituents', + '(horizontal_dimension, vertical_layer_dimension, ' + 'number_of_ccpp_constituents)', + intent='in', + ) + arg = _resolve_constituent_arg( + suite_var, 'run', hd, {}, 'consts_user', 'mysuite', + ) + self.assertIsNotNone(arg) + self.assertEqual(arg.source, 'constituent') + # Goes on the dedicated channel. + self.assertEqual(arg.used_const_dim_std_names, + {'number_of_ccpp_constituents'}) + # Does NOT leak into the host-side channels. + self.assertNotIn('number_of_ccpp_constituents', + arg.used_dim_std_names) + self.assertNotIn('number_of_ccpp_constituents', + arg.constituent_extra_symbols) + + def test_no_const_dim_when_arg_does_not_reference_it(self): + # cldliq is 2D — no framework-constituent dim refs. Uses + # _load_full_host_dict so the resolver routes through the + # constituent auto-provisioning path (Path 2: is_constituent + + # intent=in). + from generator.suite_resolver import _resolve_constituent_arg + hd = _load_full_host_dict() + suite_var = self._scheme_var( + 'cldliq', 'cloud_liquid_water_mixing_ratio', + '(horizontal_dimension, vertical_layer_dimension)', + intent='in', + ) + suite_var.set_attr('advected', 'True', _ctx()) + arg = _resolve_constituent_arg( + suite_var, 'run', hd, {}, 'cldliq_user', 'mysuite', + ) + self.assertIsNotNone(arg) + self.assertEqual(arg.used_const_dim_std_names, set()) + + def test_minimum_values_routes_to_vars_minvalue(self): + # ``ccpp_constituent_minimum_values`` is a framework-named std + # whose value is per-constituent and lives on + # ``ccpp_model_constituents_t%vars_minvalue(:)``. The resolver + # must route it through Path 1b (framework-name) — the + # ``vars_minvalue`` member, not Path 2 (constituent auto- + # provisioning). Drives cam-sima's ``qneg`` scheme: under the + # original capgen contract this was a host-USE'd module array; + # capgen exposes it through the per-instance object. + from generator.suite_resolver import _resolve_constituent_arg + hd = _load_full_host_dict() + suite_var = self._scheme_var( + 'qmin', 'ccpp_constituent_minimum_values', + '(number_of_ccpp_constituents)', + intent='in', + ) + arg = _resolve_constituent_arg( + suite_var, 'run', hd, {}, 'qneg', 'mysuite', + ) + self.assertIsNotNone(arg) + self.assertEqual(arg.source, 'constituent') + inst_local = hd['instance_number'].local_name + self.assertEqual( + arg.call_expr, + 'ccpp_model_constituents_obj({})%vars_minvalue(:)'.format( + inst_local), + ) + # number_of_ccpp_constituents goes on the dedicated channel. + self.assertEqual(arg.used_const_dim_std_names, + {'number_of_ccpp_constituents'}) + self.assertNotIn('number_of_ccpp_constituents', + arg.used_dim_std_names) + self.assertNotIn('number_of_ccpp_constituents', + arg.constituent_extra_symbols) + + +class TestIndexSymbolNameMangling(unittest.TestCase): + """``_index_symbol_name`` keeps short ``index_of_`` names + intact, but mangles overlong CAM-SIMA-style names down to the + Fortran 63-char identifier limit with a deterministic SHA hash so + every emit/reference site agrees on the symbol.""" + + def test_short_name_passes_through(self): + from generator.suite_resolver import _index_symbol_name + self.assertEqual(_index_symbol_name('water_vapor'), + 'index_of_water_vapor') + + def test_overlong_name_truncated_and_hashed(self): + from generator.suite_resolver import _index_symbol_name + long_base = ('cloud_liquid_water_mixing_ratio_' + 'wrt_moist_air_and_condensed_water') + sym = _index_symbol_name(long_base) + # Must be a Fortran-legal identifier (≤ 63 chars), prefixed + # with index_of_, and stable across calls. + self.assertLessEqual(len(sym), 63) + self.assertTrue(sym.startswith('index_of_')) + self.assertEqual(sym, _index_symbol_name(long_base)) + + def test_distinct_bases_distinct_symbols(self): + # Two CAM-SIMA constituents share the same long suffix; the + # hash component must keep their symbols distinct so the + # framework's per-constituent integer storage doesn't alias. + from generator.suite_resolver import _index_symbol_name + a = _index_symbol_name( + 'cloud_liquid_water_mixing_ratio_' + 'wrt_moist_air_and_condensed_water') + b = _index_symbol_name( + 'water_vapor_mixing_ratio_' + 'wrt_moist_air_and_condensed_water') + self.assertNotEqual(a, b) + + def test_used_in_auto_provisioned_call_expr(self): + # End-to-end check: the auto-provisioning subscript path + # (Path 2) routes the index_of_ token through the helper, + # so the call_expr that lands in the group cap is a legal + # Fortran symbol. + from generator.suite_resolver import ( + _resolve_constituent_arg, _index_symbol_name, + ) + long_base = ('cloud_liquid_water_mixing_ratio_' + 'wrt_moist_air_and_condensed_water') + hd = _load_full_host_dict() + scheme_var = self._scheme_var_for_mangling( + 'cldliq', long_base, + '(horizontal_dimension, vertical_layer_dimension)', + ) + scheme_var.set_attr('advected', 'True', _ctx()) + arg = _resolve_constituent_arg( + scheme_var, 'run', hd, {}, 'consumer', 'mysuite', + ) + self.assertIsNotNone(arg) + expected_index_sym = _index_symbol_name(long_base) + self.assertIn(expected_index_sym, arg.constituent_extra_symbols) + # The long raw form must NOT appear (would blow the Fortran limit). + self.assertNotIn('index_of_' + long_base, + arg.constituent_extra_symbols) + self.assertIn(expected_index_sym, arg.call_expr) + + @staticmethod + def _scheme_var_for_mangling(local, std_name, dims, intent='in'): + from metadata.metadata_table import MetaVar + ctx = _ctx() + v = MetaVar(local, ctx) + v.set_attr('standard_name', std_name, ctx) + v.set_attr('units', 'none', ctx) + v.set_attr('dimensions', dims, ctx) + v.set_attr('type', 'real', ctx) + v.set_attr('kind', 'kind_phys', ctx) + v.set_attr('intent', intent, ctx) + return v + + +class TestConstSubscriptHelper(unittest.TestCase): + """``_const_dim_part`` / ``_build_const_subscript``: + ``number_of_ccpp_constituents`` becomes ``':'`` and is routed + through the dedicated ``used_const_dim_std`` channel — NOT through + ``used_host_std`` (which would imply USE/dummy-arg plumbing) and + NOT through ``used_const_std`` (which would imply a USE'd symbol). + """ + + def setUp(self): + from generator.suite_resolver import ( + _const_dim_part, _build_const_subscript, + ) + self._const_dim_part = _const_dim_part + self._build_const_subscript = _build_const_subscript + self.hd = _load_full_host_dict() + + def test_dim_part_returns_colon(self): + part, used_host, used_const, used_const_dim = self._const_dim_part( + 'number_of_ccpp_constituents', 'run', self.hd, + ) + self.assertEqual(part, ':') + # NOT a host dim — _collect_group_uses / _extra_dim_ctrl_entries + # would mishandle it if it leaked here. + self.assertEqual(used_host, set()) + # NOT a USE'd symbol — it isn't a public name on + # ccpp_host_constituents in the per-instance design. + self.assertEqual(used_const, set()) + # The introspection-only channel. + self.assertEqual(used_const_dim, {'number_of_ccpp_constituents'}) + + def test_dim_part_handles_explicit_lower_bound(self): + part, used_host, used_const, used_const_dim = self._const_dim_part( + 'ccpp_constant_one:number_of_ccpp_constituents', 'run', self.hd, + ) + self.assertEqual(part, ':') + self.assertEqual(used_host, set()) + self.assertEqual(used_const, set()) + self.assertEqual(used_const_dim, {'number_of_ccpp_constituents'}) + + def test_full_3d_subscript(self): + # Mirrors apply_constituent_tendencies.meta's + # (horizontal_dimension, vertical_layer_dimension, + # number_of_ccpp_constituents). + sub, used_host, used_const, used_const_dim = self._build_const_subscript( + ['horizontal_dimension', 'vertical_layer_dimension', + 'number_of_ccpp_constituents'], + 'run', self.hd, + ) + self.assertEqual(sub, '(lb:ub, 1:nlev, :)') + # Host dims (horizontal_*, vertical_*) live in used_host; + # number_of_ccpp_constituents lives in used_const_dim. + self.assertNotIn('number_of_ccpp_constituents', used_host) + self.assertEqual(used_const, set()) + self.assertEqual(used_const_dim, {'number_of_ccpp_constituents'}) + + +# auto-clone-constituents: the entire class below exists because the +# legacy auto-clone shim introduces a second source for "this suite +# needs a per-suite ``_dynamic_constituents`` buffer". When +# the shim retires, this class can be deleted -- the property collapses +# to ``bool(self.constituent_register_calls)`` and the existing +# register-path tests elsewhere already cover that case. +class TestNeedsDynamicConstituentsBufferProperty(unittest.TestCase): + """``SuiteResolution.needs_dynamic_constituents_buffer`` is the + single source of truth for "this suite needs a + ``_dynamic_constituents`` buffer". Centralising the rule + here keeps legacy-shim state (``auto_cloned_constituents``) out of + every generator emitter; consumers reference the property instead + of OR-ing the two underlying fields. When the legacy auto-clone + shim retires, only this property's body changes.""" + + def test_false_when_neither(self): + # auto-clone-constituents: empty-state baseline. + from generator.suite_resolver import SuiteResolution + sr = SuiteResolution(suite_name='s') + self.assertFalse(sr.needs_dynamic_constituents_buffer) + + def test_true_for_register_calls(self): + # auto-clone-constituents: register-only path -- verifies the + # property still fires correctly for non-shim registrations + # after the OR-abstraction landed. + from generator.suite_resolver import SuiteResolution + sr = SuiteResolution( + suite_name='s', + constituent_register_calls=[('register_constituents', 'register')], + ) + self.assertTrue(sr.needs_dynamic_constituents_buffer) + + def test_true_for_auto_cloned_only(self): + # auto-clone-constituents: shim-only path -- regression for + # the CAM-SIMA kessler_test build (2026-06-03). + from generator.suite_resolver import SuiteResolution, AutoCloneEntry + sr = SuiteResolution( + suite_name='s', + auto_cloned_constituents=[AutoCloneEntry( + std_name='water_vapor', long_name='', diag_name='qv', + units='kg kg-1', vertical_dim='vertical_layer_dimension', + advected=True, molar_mass=0.0, default_value=None, + min_value=None, water_species=None, mixing_ratio_type=None, + )], + ) + self.assertTrue(sr.needs_dynamic_constituents_buffer) + + +class TestConstituentResolverErrors(unittest.TestCase): + """Mismatched constituent-flag + intent + std-name combinations error.""" + + _SCHEME_TEMPLATE = ( + '[ccpp-table-properties]\n' + ' name = bad_scheme\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = bad_scheme_run\n' + ' type = scheme\n' + '[ x ]\n' + ' standard_name = {std}\n' + ' units = kg kg-1\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' + ' type = real | kind = kind_phys\n' + ' intent = {intent}\n' + ' {flag} = .true.\n' + ) + + _SUITE_XML = ( + '\n' + '\n' + ' bad_scheme\n' + '\n' + ) + + def _build_store(self, std, intent, flag): + with tempfile.NamedTemporaryFile('w', suffix='.meta', delete=False) as fh: + fh.write(self._SCHEME_TEMPLATE.format( + std=std, intent=intent, flag=flag, + )) + path = fh.name + try: + tables = parse_metadata_file(path) + finally: + os.unlink(path) + return SchemeStore.build_from(tables) + + def _resolve(self, std, intent, flag): + hd = _load_constituent_host_dict() + store = self._build_store(std, intent, flag) + with tempfile.TemporaryDirectory() as tmpdir: + xml = os.path.join(tmpdir, 's.xml') + with open(xml, 'w') as f: + f.write(self._SUITE_XML) + from generator.suite_xml import parse_suite_xml + import logging + suite = parse_suite_xml(xml, tmpdir, logging.getLogger('t'), + skip_validation=True) + return resolve_suite(suite, store, hd) + + def test_base_constituent_intent_out_raises(self): + # advected + intent=out on a non-tendency std name → reject. + with self.assertRaises(CCPPError) as ctx: + self._resolve('air_temperature_extra', 'out', 'advected') + self.assertIn("'tendency_of_'", str(ctx.exception)) + + def test_tendency_intent_in_resolves_to_tendency_array(self): + # Rule (b): a constituent tendency may be CONSUMED (e.g. a diagnostics + # scheme). intent=in on a tendency_of_* std name now reads the framework + # tendency column instead of erroring. + res = self._resolve('tendency_of_air_temperature', 'in', 'constituent') + arg = list(iter_phase_calls(res.groups[0].phase_calls['run']))[0].args[0] + self.assertEqual(arg.source, 'constituent') + self.assertIn('vars_layer_tend', arg.call_expr) + + def test_tendency_intent_inout_resolves_to_tendency_array(self): + res = self._resolve('tendency_of_air_temperature', 'inout', 'constituent') + arg = list(iter_phase_calls(res.groups[0].phase_calls['run']))[0].args[0] + self.assertEqual(arg.source, 'constituent') + self.assertIn('vars_layer_tend', arg.call_expr) + + +class TestConstituentConsumerInferenceRuleB(unittest.TestCase): + """Rule (b): an UNFLAGGED scheme may CONSUME a constituent that another + scheme flags -- a base constituent (``advected``) read via ``vars_layer``, or + a constituent tendency (``constituent`` on a ``tendency_of_*`` producer) read + via ``vars_layer_tend``. The consumer infers it from the scheme-wide flag + set and never re-flags. Whether a name is a constituent is the host's call, + so a name NO scheme flags stays an ordinary variable. + """ + + _PRODUCER_CONSUMER = ( + '[ccpp-table-properties]\n' + ' name = tend_producer\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = tend_producer_run\n' + ' type = scheme\n' + '[ qt ]\n' + ' standard_name = tendency_of_air_temperature\n' + ' units = K s-1\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' + ' type = real | kind = kind_phys\n' + ' intent = out\n' + ' constituent = .true.\n' + '\n' + '[ccpp-table-properties]\n' + ' name = tend_consumer\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = tend_consumer_run\n' + ' type = scheme\n' + '[ qt ]\n' + ' standard_name = tendency_of_air_temperature\n' + ' units = K s-1\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' + ' type = real | kind = kind_phys\n' + ' intent = in\n' + ) + + # Consumer only (no scheme flags the tendency as a constituent). + _CONSUMER_ONLY = ( + '[ccpp-table-properties]\n' + ' name = tend_consumer\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = tend_consumer_run\n' + ' type = scheme\n' + '[ qt ]\n' + ' standard_name = tendency_of_air_temperature\n' + ' units = K s-1\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' + ' type = real | kind = kind_phys\n' + ' intent = in\n' + ) + + # A base constituent (advected) flagged by one scheme and consumed unflagged + # by another; nothing else provides it. + _BASE_FLAGGED_AND_UNFLAGGED = ( + '[ccpp-table-properties]\n' + ' name = base_flagged\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = base_flagged_run\n' + ' type = scheme\n' + '[ q ]\n' + ' standard_name = made_up_dry_mixing_ratio\n' + ' units = kg kg-1\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' + ' type = real | kind = kind_phys\n' + ' intent = in\n' + ' advected = .true.\n' + '\n' + '[ccpp-table-properties]\n' + ' name = base_unflagged\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = base_unflagged_run\n' + ' type = scheme\n' + '[ q ]\n' + ' standard_name = made_up_dry_mixing_ratio\n' + ' units = kg kg-1\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' + ' type = real | kind = kind_phys\n' + ' intent = in\n' + ) + + def _resolve(self, meta, schemes): + hd = _load_constituent_host_dict() + with tempfile.NamedTemporaryFile('w', suffix='.meta', delete=False) as fh: + fh.write(meta) + path = fh.name + try: + store = SchemeStore.build_from(parse_metadata_file(path)) + finally: + os.unlink(path) + xml = ('\n' + '\n' + ' \n' + + ''.join(' {}\n'.format(s) for s in schemes) + + ' \n\n') + with tempfile.TemporaryDirectory() as tmpdir: + xpath = os.path.join(tmpdir, 's.xml') + with open(xpath, 'w') as f: + f.write(xml) + from generator.suite_xml import parse_suite_xml + import logging + suite = parse_suite_xml(xpath, tmpdir, logging.getLogger('t'), + skip_validation=True) + return resolve_suite(suite, store, hd) + + def test_unflagged_consumer_reads_tendency_array(self): + res = self._resolve(self._PRODUCER_CONSUMER, + ['tend_producer', 'tend_consumer']) + calls = list(iter_phase_calls(res.groups[0].phase_calls['run'])) + producer_arg = calls[0].args[0] + consumer_arg = calls[1].args[0] + self.assertEqual(producer_arg.source, 'constituent') + self.assertIn('vars_layer_tend', producer_arg.call_expr) + # The consumer carries NO constituent flag, yet routes to the SAME + # framework tendency column (rule b inference from the producer). + self.assertEqual(consumer_arg.source, 'constituent') + self.assertIn('vars_layer_tend', consumer_arg.call_expr) + + def test_unflagged_tendency_without_producer_is_not_constituent(self): + # Nothing flags the tendency, so it stays an ordinary variable: an + # unprovided consumer is a normal "not provided" error, never silently + # routed to the constituent tendency array. + with self.assertRaises(CCPPError) as ctx: + self._resolve(self._CONSUMER_ONLY, ['tend_consumer']) + self.assertIn('not provided', str(ctx.exception)) + + def test_unflagged_base_consumer_reads_vars_layer(self): + # Symmetric to the tendency case: a base constituent flagged advected by + # one scheme is read by an unflagged consumer via the SAME base column + # (vars_layer, NOT the tendency array). + res = self._resolve(self._BASE_FLAGGED_AND_UNFLAGGED, + ['base_flagged', 'base_unflagged']) + calls = list(iter_phase_calls(res.groups[0].phase_calls['run'])) + flagged = calls[0].args[0] + unflagged = calls[1].args[0] + self.assertEqual(flagged.source, 'constituent') + self.assertIn('%vars_layer(', flagged.call_expr) + self.assertEqual(unflagged.source, 'constituent') + self.assertIn('%vars_layer(', unflagged.call_expr) + self.assertNotIn('vars_layer_tend', unflagged.call_expr) + + +class TestHostDeclaredIndexOfWinsOverConstituents(unittest.TestCase): + """Regression: when the host declares an ``index_of_`` integer as a + regular host variable, scheme references to that std_name must resolve + to the host's short local name — NOT route through the constituent + auto-provisioning path (which would emit a parallel module-level + integer named after the full std_name in ``ccpp_host_constituents`` + and, for SCM-style long std_names, blow the Fortran 63-char identifier + limit). + """ + + _HOST_SRC = ( + '[ccpp-table-properties]\n' + ' name = scm_host_mod\n' + ' type = host\n' + '[ccpp-arg-table]\n' + ' name = scm_host_mod\n' + ' type = host\n' + # Mirrors SCM's GFS_typedefs: short Fortran local name `ntcw` + # paired with a long index_of_* standard name. + '[ ntcw ]\n' + ' standard_name = index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array\n' + ' units = index\n' + ' type = integer\n' + ' protected = True\n' + ' dimensions = ()\n' + ) + + def _scheme_var(self, local, std_name, intent='in'): + from metadata.metadata_table import MetaVar + ctx = _ctx() + v = MetaVar(local, ctx) + v.set_attr('standard_name', std_name, ctx) + v.set_attr('units', 'index', ctx) + v.set_attr('dimensions', '()', ctx) + v.set_attr('type', 'integer', ctx) + v.set_attr('intent', intent, ctx) + return v + + def test_host_index_of_resolves_to_host_local_name(self): + hd = build_flat_host_dict(_parse(self._HOST_SRC), [], []) + suite_var = self._scheme_var( + 'ntcw', + 'index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array', + intent='in', + ) + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'gfs_mp_generic_pre', set()) + # Host metadata wins: source=host, short local name, NO leakage of + # the long std_name into ccpp_host_constituents. + self.assertEqual(arg.source, 'host') + self.assertEqual(arg.call_expr, 'ntcw') + self.assertIsNotNone(arg.host_entry) + self.assertEqual(arg.host_entry.local_name, 'ntcw') + + def test_unclaimed_index_of_still_routes_to_constituents(self): + """The framework auto-provisioning path is preserved for + ``index_of_`` names the host does NOT declare — required for + capgen-owned constituent flows (cf. the advection e2e test).""" + hd = build_flat_host_dict(_parse(self._HOST_SRC), [], []) + suite_var = self._scheme_var( + 'idx_other', 'index_of_some_other_constituent_not_in_host', + intent='in', + ) + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'some_scheme', set()) + self.assertEqual(arg.source, 'constituent') + self.assertEqual(arg.call_expr, + 'index_of_some_other_constituent_not_in_host') + + +class TestDimDDTComponentResolution(unittest.TestCase): + """When a dimension standard name maps to a DDT-component host + entry (e.g. ``vertical_layer_dimension`` is ``physics%Model%levs``, + a two-level DDT walk inside the SCM/UFS host), the resolver must: + + 1. Emit the full ``access_path`` in subscript expressions + (``1:physics%Model%levs``), NOT the bare leaf (``1:levs``). + 2. Walk back to the access-path *root* for the USE statement + (``use scm_host_mod, only: physics``), NOT the leaf + (``use scm_host_mod, only: levs`` — undefined symbol). + 3. Behave identically to today for plain module-level host vars + (``access_path == local_name``). + + Historical: this was a long-standing pain point in the original + capgen → SCM migration. The pre-2026-05-13 capgen emitted + ``use scm_type_defs, only: levs`` and similar bogus imports for + every DDT-component dim, producing many ``Symbol referenced ... + not found in module`` errors at compile time. + """ + + # Mimics SCM: ``physics`` is host-level (module ``scm_host_mod``), + # of type ``physics_t`` (a DDT) which has component ``Model`` of + # type ``gfs_control_t`` (a DDT) which has scalar ``levs`` and + # ``ncols`` declared on it. + _DDT_SRC = ( + '[ccpp-table-properties]\n' + ' name = gfs_control_t\n' + ' type = ddt\n' + '[ccpp-arg-table]\n' + ' name = gfs_control_t\n' + ' type = ddt\n' + '[levs]\n' + ' standard_name = vertical_layer_dimension\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + '[ncol]\n' + ' standard_name = horizontal_dimension_total\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + '\n' + '[ccpp-table-properties]\n' + ' name = physics_t\n' + ' type = ddt\n' + '[ccpp-arg-table]\n' + ' name = physics_t\n' + ' type = ddt\n' + '[Model]\n' + ' standard_name = gfs_control_instance\n' + ' units = DDT\n' + ' dimensions = ()\n' + ' type = gfs_control_t\n' + ) + + _HOST_SRC = ( + '[ccpp-table-properties]\n' + ' name = scm_host_mod\n' + ' type = host\n' + '[ccpp-arg-table]\n' + ' name = scm_host_mod\n' + ' type = host\n' + # Plain (non-DDT) horizontal dim — for the mixed-dim test. + '[ncols]\n' + ' standard_name = horizontal_dimension\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + # The DDT instance. Resolver should produce + # access_path == 'physics' for the instance, and + # 'physics%Model%levs' for the leaf dim. + '[physics]\n' + ' standard_name = physics_state_instance\n' + ' units = DDT\n' + ' dimensions = ()\n' + ' type = physics_t\n' + ) + + def _host_dict(self): + return build_flat_host_dict( + _parse(self._HOST_SRC), [], _parse(self._DDT_SRC), + ) + + # ---- Unit-level: host_dict entry shape (the prerequisite) ----------- + + def test_host_dict_levs_has_ddt_walk_access_path(self): + hd = self._host_dict() + entry = hd.get('vertical_layer_dimension') + self.assertIsNotNone(entry) + self.assertEqual(entry.local_name, 'levs') + self.assertEqual(entry.access_path, 'physics%Model%levs') + self.assertEqual(entry.module_name, 'scm_host_mod') + + # ---- Unit-level: _resolve_single_bound (the primary fix site) ------- + + def test_resolve_single_bound_returns_access_path_for_ddt_dim(self): + from generator.suite_resolver import _resolve_single_bound + hd = self._host_dict() + used = set() + result = _resolve_single_bound('vertical_layer_dimension', hd, used) + self.assertEqual(result, 'physics%Model%levs') + self.assertIn('vertical_layer_dimension', used) + + def test_resolve_single_bound_plain_var_unchanged(self): + # ncols is a plain host var (access_path == local_name == 'ncols'). + from generator.suite_resolver import _resolve_single_bound + hd = self._host_dict() + used = set() + result = _resolve_single_bound('horizontal_dimension', hd, used) + self.assertEqual(result, 'ncols') + + # ---- Unit-level: _one_dim_part wrapping ----------------------------- + + def test_one_dim_part_bare_form_uses_ddt_walk(self): + from generator.suite_resolver import _one_dim_part + hd = self._host_dict() + # Bare std name normalises to ``ccpp_constant_one:``. + part, used = _one_dim_part( + 'vertical_layer_dimension', 'run', hd, + ) + self.assertEqual(part, '1:physics%Model%levs') + self.assertIn('vertical_layer_dimension', used) + + def test_one_dim_part_range_form_uses_ddt_walk(self): + from generator.suite_resolver import _one_dim_part + hd = self._host_dict() + part, used = _one_dim_part( + 'ccpp_constant_one:vertical_layer_dimension', 'run', hd, + ) + self.assertEqual(part, '1:physics%Model%levs') + + def test_one_dim_part_ddt_dim_as_lower_bound(self): + # Stress: DDT-component appears as the LOWER bound of a range. + from generator.suite_resolver import _one_dim_part + hd = self._host_dict() + part, used = _one_dim_part( + 'vertical_layer_dimension:horizontal_dimension_total', 'run', hd, + ) + # Both bounds walked: physics%Model%levs : physics%Model%ncol + self.assertEqual(part, 'physics%Model%levs:physics%Model%ncol') + + # ---- Unit-level: _build_call_subscript composition ----------------- + + def test_build_call_subscript_two_ddt_dims(self): + # Two DDT-component dims side by side: both must walk. + from generator.suite_resolver import _build_call_subscript + hd = self._host_dict() + sub, used = _build_call_subscript( + ['horizontal_dimension_total', 'vertical_layer_dimension'], + 'run', hd, + ) + self.assertEqual( + sub, '(1:physics%Model%ncol, 1:physics%Model%levs)', + ) + + def test_build_call_subscript_pure_ddt_dims(self): + from generator.suite_resolver import _build_call_subscript + hd = self._host_dict() + sub, used = _build_call_subscript( + ['vertical_layer_dimension', 'vertical_layer_dimension'], + 'run', hd, + ) + self.assertEqual(sub, '(1:physics%Model%levs, 1:physics%Model%levs)') + + # ---- Sliced-subscript path: _build_merged_subscript --------------- + + def test_build_merged_subscript_ddt_index_token(self): + # Mirrors host metadata like ``q(:,:,vertical_layer_dimension)`` + # but here we exercise the helper directly with a synthetic + # local_subscript carrying a std-name token whose target lives + # on a DDT. The third token must be the full DDT walk, not + # the bare leaf — bug pre-2026-05-13 produced ``levs`` instead + # of ``physics%Model%levs`` and the generated cap then failed + # to compile against the host module. + from generator.suite_resolver import _build_merged_subscript + hd = self._host_dict() + # Use ``horizontal_dimension_total`` for the leading dims so the + # fixture doesn't need loop bounds. + merged, used = _build_merged_subscript( + host_dims=['horizontal_dimension_total', + 'horizontal_dimension_total'], + local_subscript=[':', ':', 'vertical_layer_dimension'], + phase='run', host_dict=hd, suite_vars={}, + ) + # Third subscript token = full DDT walk. + self.assertTrue( + merged.rstrip(')').endswith('physics%Model%levs'), + 'merged subscript should end with the DDT walk; got {!r}' + .format(merged), + ) + # Leaf should not leak. + self.assertNotIn(', levs)', merged) + + # ---- Group-cap USE collection: _collect_dim_uses -------------------- + + @staticmethod + def _mock_arg(used_dim_std_names): + from unittest.mock import MagicMock + arg = MagicMock() + arg.used_dim_std_names = set(used_dim_std_names) + return arg + + def _mock_rg(self, arg): + from unittest.mock import MagicMock + from generator.suite_resolver import ResolvedCall + resolved_group = MagicMock() + resolved_group.phase_calls = { + 'run': [ResolvedCall( + scheme_name='dummy', phase='run', + args=[arg], scheme_module='dummy_mod', + )], + } + return resolved_group + + def test_collect_dim_uses_walks_to_root_for_ddt_dim(self): + from generator.suite_resolver import _collect_dim_uses + hd = self._host_dict() + arg = self._mock_arg({'vertical_layer_dimension'}) + resolved_group = self._mock_rg(arg) + dim_uses = _collect_dim_uses(resolved_group, hd, suite_vars={}) + # USE clause must pull the ROOT (``physics``), not the leaf + # (``levs``, which is not a module symbol). + self.assertIn('scm_host_mod', dim_uses) + self.assertIn('physics', dim_uses['scm_host_mod']) + self.assertNotIn('levs', dim_uses['scm_host_mod']) + self.assertNotIn('Model', dim_uses['scm_host_mod']) + + def test_collect_dim_uses_plain_var_unchanged(self): + # Regression: behaviour identical for plain host vars. + from generator.suite_resolver import _collect_dim_uses + hd = self._host_dict() + arg = self._mock_arg({'horizontal_dimension'}) + resolved_group = self._mock_rg(arg) + dim_uses = _collect_dim_uses(resolved_group, hd, suite_vars={}) + self.assertEqual(dim_uses.get('scm_host_mod'), {'ncols'}) + + def test_collect_dim_uses_two_ddt_dims_dedupe_root(self): + # Both ``vertical_layer_dimension`` and + # ``horizontal_dimension_total`` live on the same ``physics`` + # instance — USE clause emits ``physics`` ONCE, not twice. + from generator.suite_resolver import _collect_dim_uses + hd = self._host_dict() + arg = self._mock_arg({'vertical_layer_dimension', + 'horizontal_dimension_total'}) + resolved_group = self._mock_rg(arg) + dim_uses = _collect_dim_uses(resolved_group, hd, suite_vars={}) + self.assertEqual(dim_uses.get('scm_host_mod'), {'physics'}) + + # ---- End-to-end: resolve_suite + group cap output ------------------ + + def test_end_to_end_ddt_dim_in_group_cap(self): + """Scheme arg with a DDT-component dim should compile. + + Builds the full pipeline: parse scheme + host metadata, resolve + the suite, and generate the group cap text. Asserts: + + - The scheme call subscript contains ``1:physics%Model%levs`` + (NOT ``1:levs``). + - The group cap's ``use scm_host_mod, only: ...`` clause + contains ``physics`` (the DDT root) and NOT ``levs`` (the + leaf, which would be an undefined symbol). + """ + # Add control vars so the static API can build. + control_src = ( + '[ccpp-table-properties]\n' + ' name = control\n' + ' type = control\n' + '[ccpp-arg-table]\n' + ' name = control\n' + ' type = control\n' + '[errmsg]\n' + ' standard_name = ccpp_error_message\n' + ' units = none\n' + ' dimensions = ()\n' + ' type = character | kind = len=512\n' + '[errflg]\n' + ' standard_name = ccpp_error_code\n' + ' units = 1\n' + ' dimensions = ()\n' + ' type = integer\n' + '[ilb]\n' + ' standard_name = horizontal_loop_begin\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + '[iub]\n' + ' standard_name = horizontal_loop_end\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + ) + scheme_src = ( + '[ccpp-table-properties]\n' + ' name = ddt_dim_user\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = ddt_dim_user_run\n' + ' type = scheme\n' + '[temp]\n' + ' standard_name = air_temperature\n' + ' units = K\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' + ' type = real | kind = kind_phys\n' + ' intent = inout\n' + '[errmsg]\n' + ' standard_name = ccpp_error_message\n' + ' units = none\n' + ' dimensions = ()\n' + ' type = character | kind = len=*\n' + ' intent = out\n' + '[errflg]\n' + ' standard_name = ccpp_error_code\n' + ' units = 1\n' + ' dimensions = ()\n' + ' type = integer\n' + ' intent = out\n' + ) + # Extend the host with horizontal_dimension + air_temperature + # so the scheme's dims resolve. + host_src = self._HOST_SRC + ( + '[gt0]\n' + ' standard_name = air_temperature\n' + ' units = K\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' + ' type = real | kind = kind_phys\n' + ) + hd = build_flat_host_dict( + _parse(host_src), _parse(control_src), _parse(self._DDT_SRC), + ) + store = SchemeStore.build_from(_parse(scheme_src)) + + # Build a tiny suite XML and resolve it. + suite_xml = ( + '\n' + '\n' + ' ddt_dim_user\n' + '\n' + ) + with tempfile.TemporaryDirectory() as tmpdir: + xml_path = os.path.join(tmpdir, 's.xml') + with open(xml_path, 'w') as fh: + fh.write(suite_xml) + from generator.suite_xml import parse_suite_xml + import logging + logger = logging.getLogger('ddt_dim_e2e') + suite = parse_suite_xml(xml_path, tmpdir, logger, + skip_validation=True) + suite_resolution = resolve_suite(suite, store, hd) + + # Inspect the resolved scheme arg's subscript. + run_call = list(iter_phase_calls(suite_resolution.groups[0].phase_calls['run']))[0] + temp_arg = [a for a in run_call.args + if a.scheme_local_name == 'temp'][0] + self.assertIn('physics%Model%levs', temp_arg.subscript) + # Leaf name must not appear bare anywhere in the call expr. + self.assertNotIn(', 1:levs)', temp_arg.call_expr) + + # Now emit the group cap and inspect the USE clause. + from generator.group_cap import _generate_group_cap + group_lines = _generate_group_cap( + suite_name=suite_resolution.suite_name, + group_name=suite_resolution.groups[0].group_name, + resolved_group=suite_resolution.groups[0], host_dict=hd, + ) + group_text = '\n'.join(group_lines) + # Pull the host-module USE line. Must import ``physics``, + # must NOT import the bare leaf ``levs``. + import re as _re + host_use_lines = [ + ln for ln in group_text.splitlines() + if _re.search(r'use\s+scm_host_mod\b', ln) + ] + self.assertTrue(host_use_lines, + "group cap missing ``use scm_host_mod`` line") + joined = ' '.join(host_use_lines) + self.assertIn('physics', joined) + # Word-boundary check so we don't false-match a substring like + # ``physics`` containing ``levs`` etc. + self.assertIsNone(_re.search(r'\blevs\b', joined), + "group cap leaked DDT-leaf ``levs`` into USE: " + + joined) + self.assertIsNone(_re.search(r'\bModel\b', joined), + "group cap leaked DDT-mid-component ``Model`` " + "into USE: " + joined) + + +######################################################################## +# Doctest loader +######################################################################## + +######################################################################## +# validate_init_dimensions: a non-allocatable suite var dimensioned by a +# scheme-updated-after-register quantity must be allocatable. +######################################################################## + +class TestValidateInitDimensions(unittest.TestCase): + """Reject a non-allocatable suite var whose dimension is written by a + scheme in a phase after register (capgen can't size it at init). + Regression for the rrtmgp pint_day OOM.""" + + @staticmethod + def _arg(std, intent): + return types.SimpleNamespace(standard_name=std, intent=intent) + + @classmethod + def _sr(cls, phase_writes, suite_vars): + # phase_writes: {phase: [(std, intent), ...]}; suite_vars: [(std, alloc, dims)] + calls = [] + pc = {} + for phase, writes in phase_writes.items(): + pc[phase] = [ResolvedCall(scheme_name='s', phase=phase, + args=[cls._arg(s, i) for s, i in writes])] + grp = types.SimpleNamespace(phase_calls=pc) + svs = {s: types.SimpleNamespace(standard_name=s, allocatable=a, dimensions=d) + for s, a, d in suite_vars} + return types.SimpleNamespace(groups=[grp], suite_vars=svs) + + def test_late_written_dim_nonalloc_raises(self): + sr = self._sr( + {'timestep_init': [('mydim', 'out')], 'run': [('myarr', 'out')]}, + [('mydim', False, []), ('myarr', False, ['mydim'])], + ) + with self.assertRaisesRegex(CCPPError, "myarr.*mydim|allocatable"): + validate_init_dimensions(sr) + + def test_register_written_dim_ok(self): + sr = self._sr( + {'register': [('mydim', 'out')], 'run': [('myarr', 'out')]}, + [('mydim', False, []), ('myarr', False, ['mydim'])], + ) + validate_init_dimensions(sr) # no raise + + def test_allocatable_var_skipped(self): + sr = self._sr( + {'timestep_init': [('mydim', 'out')], 'run': [('myarr', 'out')]}, + [('mydim', False, []), ('myarr', True, ['mydim'])], + ) + validate_init_dimensions(sr) # allocatable -> not capgen's to size + + def test_host_static_dim_ok(self): + # 'hostdim' is never written by a scheme -> assumed available at init. + sr = self._sr( + {'run': [('myarr', 'out')]}, + [('myarr', False, ['hostdim'])], + ) + validate_init_dimensions(sr) # no raise + + def test_range_dimension_token_checked(self): + sr = self._sr( + {'timestep_init': [('mydim', 'inout')], 'run': [('myarr', 'out')]}, + [('mydim', False, []), ('myarr', False, ['ccpp_constant_one:mydim'])], + ) + with self.assertRaises(CCPPError): + validate_init_dimensions(sr) + + +def load_tests(loader, tests, ignore): + import generator.suite_resolver as suite_resolution + import generator.group_cap as gc + tests.addTests(doctest.DocTestSuite(suite_resolution)) + tests.addTests(doctest.DocTestSuite(gc)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_suite_types.py b/unit-tests/test_suite_types.py new file mode 100644 index 00000000..5a2fedfd --- /dev/null +++ b/unit-tests/test_suite_types.py @@ -0,0 +1,354 @@ +"""Tests for generator.suite_types — pointer-wrapper types module emitter. + +Focuses on the type-clause builder and DDT USE emission. Doctests on +the helpers themselves cover the small cases; this file exercises the +end-to-end ``_generate_suite_types`` output structure. +""" + +import unittest + +from types import SimpleNamespace + +from generator.suite_types import ( + _collect_ddt_uses, + _fortran_type_str_simple, + _generate_suite_types, + _ptr_type_name, + _ptr_type_name_for_arg, +) +from metadata.parse_tools import CCPPError + + +class TestFortranTypeStrSimple(unittest.TestCase): + + def test_intrinsic_with_kind(self): + self.assertEqual( + _fortran_type_str_simple('real', 'kind_phys'), + 'real(kind=kind_phys)', + ) + + def test_intrinsic_no_kind(self): + self.assertEqual(_fortran_type_str_simple('integer', ''), 'integer') + + def test_character_len(self): + self.assertEqual( + _fortran_type_str_simple('character', 'len=512'), + 'character(len=512)', + ) + + def test_ddt_gets_type_wrapper(self): + """The bug: a bare ``cmpfsw_type, pointer :: ptr(:)`` is invalid + Fortran. Must emit ``type(cmpfsw_type)``.""" + self.assertEqual( + _fortran_type_str_simple('cmpfsw_type', ''), + 'type(cmpfsw_type)', + ) + + def test_external_gets_type_wrapper(self): + self.assertEqual( + _fortran_type_str_simple('external:mpi_f08:mpi_comm', ''), + 'type(mpi_comm)', + ) + + def test_ddt_ignores_kind(self): + """Kind is meaningless on DDTs (no kind parameters) — silently + dropped rather than emitted as ``type(foo)(kind=...)``.""" + self.assertEqual( + _fortran_type_str_simple('my_ddt', 'kind_phys'), + 'type(my_ddt)', + ) + + +class TestPtrTypeName(unittest.TestCase): + + def test_intrinsic(self): + self.assertEqual( + _ptr_type_name('real', 'kind_phys', 1), + 'real_kind_phys_rank1_ptr_type', + ) + + def test_ddt_unchanged(self): + self.assertEqual( + _ptr_type_name('cmpfsw_type', '', 1), + 'cmpfsw_type_rank1_ptr_type', + ) + + def test_external_drops_module_prefix(self): + """External types should produce a wrapper name keyed on the + typename only — otherwise it would carry colons (illegal in a + Fortran identifier).""" + self.assertEqual( + _ptr_type_name('external:mpi_f08:mpi_comm', '', 0), + 'mpi_comm_rank0_ptr_type', + ) + + def test_character_len_is_part_of_wrapper_name(self): + """Two ``character`` arguments of different lengths must produce + DIFFERENT wrapper types — Fortran disallows ``character(len=*)`` + as a DDT component, so the wrapper must bake in the literal + length. Regression for the SCM build failure where + ``character(len=10)`` and ``character(len=3)`` both got the + same ``character_rank1_ptr_type`` name and the compiler rejected + the second declaration as a duplicate.""" + n10 = _ptr_type_name('character', 'len=10', 1) + n3 = _ptr_type_name('character', 'len=3', 1) + self.assertEqual(n10, 'character_len10_rank1_ptr_type') + self.assertEqual(n3, 'character_len3_rank1_ptr_type') + self.assertNotEqual(n10, n3) + + def test_character_len_deferred(self): + """``character(len=:), allocatable`` / ``pointer`` (deferred- + length string) is legal as a DDT component when paired with + ``pointer``. The wrapper name's len suffix must NOT contain + ``:`` (illegal Fortran identifier char); emit ``_deferred``.""" + self.assertEqual( + _ptr_type_name('character', 'len=:', 1), + 'character_len_deferred_rank1_ptr_type', + ) + + def test_character_len_parameter_symbol(self): + """``len=MY_LEN`` (a Fortran parameter constant) is a valid + identifier — keep it verbatim in the wrapper name.""" + self.assertEqual( + _ptr_type_name('character', 'len=MY_LEN', 1), + 'character_lenMY_LEN_rank1_ptr_type', + ) + + def test_character_len_assumed_rejected(self): + """``character(len=*)`` cannot appear as a DDT component, so + capgen cannot synthesise a wrapper. Error must explain that + and suggest using a concrete length or deferred-length.""" + with self.assertRaisesRegex(CCPPError, 'cannot appear as a DDT component'): + _ptr_type_name('character', 'len=*', 1) + + def test_character_len_expression_rejected(self): + """Length specs that aren't identifiers (e.g. ``N+1``) can't be + encoded in a Fortran type name — error rather than emit an + illegal identifier.""" + with self.assertRaisesRegex(CCPPError, 'Fortran-identifier-safe'): + _ptr_type_name('character', 'len=N+1', 1) + + def test_error_context_prefixed_when_provided(self): + """When ``context`` is supplied, the prefix lands at the very + start of the error message so the user can immediately tell + which arg/scheme is the offender.""" + import re as _re + prefix = "scheme 'GFS_rrtmgp_pre', optional argument [foo]" + with self.assertRaisesRegex(CCPPError, _re.escape(prefix)): + _ptr_type_name('character', 'len=*', 1, context=prefix) + + def test_error_no_context_unchanged(self): + """No context → no prefix; existing message wording unchanged.""" + with self.assertRaisesRegex(CCPPError, '^character\\(len=\\*\\)'): + _ptr_type_name('character', 'len=*', 1) + + +class TestPtrTypeNameForArg(unittest.TestCase): + """``_ptr_type_name_for_arg`` builds a rich-context wrapper around + ``_ptr_type_name`` so the user can locate the offending metadata + block without grepping.""" + + def _make_arg(self, type_, host_kind, dimensions, local='foo', + std='some_standard_name', intent='in', + scheme_kind=''): + # Minimal duck-typed ResolvedArg. _ptr_type_for_arg reads + # type_/host_kind from host_entry; scheme_kind from + # arg.kind_scheme; rank from host_entry.dimensions. The + # context-string builder reads standard_name / + # scheme_local_name / intent. + host_entry = SimpleNamespace( + type=type_, kind=host_kind, dimensions=dimensions, + ) + return SimpleNamespace( + host_entry=host_entry, + suite_var=None, + kind_scheme=scheme_kind, + standard_name=std, + scheme_local_name=local, + intent=intent, + ) + + def test_clean_case_returns_name(self): + """Happy path: concrete-length character with matching scheme + kind → wrapper name built without raising.""" + arg = self._make_arg('character', 'len=10', ['ncols'], + scheme_kind='len=10') + self.assertEqual( + _ptr_type_name_for_arg(arg, 'my_scheme'), + 'character_len10_rank1_ptr_type', + ) + + def test_scheme_lenstar_host_concrete_uses_host_kind(self): + """The SCM case driving the narrow fix: scheme metadata + declares ``kind=len=*`` (legal as an assumed-length dummy) and + the host declares ``kind=len=128``. The resolver treats this + pair as compatible with no kind transform, so the pointer + wrapper must use the host's concrete length — not the + scheme's ``len=*`` (which would be illegal as a DDT + component). Without the override this raises CCPPError; with + the override it returns the concrete wrapper name.""" + arg = self._make_arg( + 'character', 'len=128', + ['number_of_active_gases_used_by_RRTMGP'], + local='active_gases_array', + std='list_of_active_gases_used_by_RRTMGP', + intent='in', + scheme_kind='len=*', + ) + self.assertEqual( + _ptr_type_name_for_arg(arg, 'GFS_rrtmgp_pre'), + 'character_len128_rank1_ptr_type', + ) + + def test_scheme_lenstar_host_deferred_uses_host_kind(self): + """Same override, but with the host declaring ``kind=len=:`` + (deferred length). The wrapper takes the host's ``len=:`` and + the name builder maps it to ``_deferred`` (existing + deferred-length rule).""" + arg = self._make_arg( + 'character', 'len=:', ['ncols'], + scheme_kind='len=*', + ) + self.assertEqual( + _ptr_type_name_for_arg(arg, 'my_scheme'), + 'character_len_deferred_rank1_ptr_type', + ) + + def test_real_kind_transform_unchanged(self): + """Regression: the narrow override applies only to + ``character`` + scheme-``len=*``. Real/integer args with a + kind transform must still take the scheme's kind so the Case-4 + transform-temp wrapping keeps working.""" + arg = self._make_arg( + 'real', 'kind_dbl_prec', ['ncols'], + scheme_kind='kind_phys', + ) + self.assertEqual( + _ptr_type_name_for_arg(arg, 'my_scheme'), + 'real_kind_phys_rank1_ptr_type', + ) + + def test_host_actually_lenstar_still_errors(self): + """Edge case: if the *host* metadata itself declares + ``kind=len=*`` (which would normally be rejected upstream), + the wrapper builder still has nothing it can use — the error + fires and names the offending scheme + arg. This guards the + error-enrichment path itself.""" + arg = self._make_arg( + 'character', 'len=*', + ['number_of_active_gases_used_by_RRTMGP'], + local='active_gases_array', + std='list_of_active_gases_used_by_RRTMGP', + intent='in', + scheme_kind='', + ) + with self.assertRaises(CCPPError) as cm: + _ptr_type_name_for_arg(arg, 'GFS_rrtmgp_pre') + msg = str(cm.exception) + self.assertIn("scheme 'GFS_rrtmgp_pre'", msg) + self.assertIn('[active_gases_array]', msg) + self.assertIn('list_of_active_gases_used_by_RRTMGP', msg) + self.assertIn('intent=in', msg) + self.assertIn('cannot appear as a DDT component', msg) + + +class TestCollectDdtUses(unittest.TestCase): + + def test_intrinsics_skipped(self): + combos = {('real', 'kind_phys', 1), ('integer', '', 0)} + self.assertEqual(_collect_ddt_uses(combos, {}), {}) + + def test_ddt_grouped_by_module(self): + combos = {('cmpfsw_type', '', 1), ('cmpfsw_type', '', 2)} + ddt_map = {'cmpfsw_type': 'module_radsw_parameters'} + uses = _collect_ddt_uses(combos, ddt_map) + self.assertEqual(uses, {'module_radsw_parameters': {'cmpfsw_type'}}) + + def test_external_module_parsed_from_type(self): + combos = {('external:mpi_f08:mpi_comm', '', 0)} + uses = _collect_ddt_uses(combos, None) + self.assertEqual(uses, {'mpi_f08': {'mpi_comm'}}) + + def test_missing_ddt_in_map_raises(self): + combos = {('some_ddt', '', 1)} + with self.assertRaisesRegex(CCPPError, "module_name"): + _collect_ddt_uses(combos, {}) + + +class TestGenerateSuiteTypesIncludesDdtUse(unittest.TestCase): + """End-to-end at the source-line level: an optional DDT-typed + argument's pointer wrapper must come with the matching ``use`` so + ``type()`` resolves at compile time, AND the declaration must + be wrapped (``type(), pointer`` — not bare ``, pointer``). + Regression for the SCM build failure on + ``ccpp_SCM_GFS_v17_p8_types.F90`` where ``cmpfsw_type, pointer :: + ptr(:)`` was emitted bare. + """ + + def test_ddt_pointer_wrapper_well_formed(self): + combos = {('cmpfsw_type', '', 1)} + ddt_map = {'cmpfsw_type': 'module_radsw_parameters'} + lines = _generate_suite_types('demo', combos, ddt_map) + text = '\n'.join(lines) + self.assertIn( + 'use module_radsw_parameters, only: cmpfsw_type', text, + "DDT USE line missing — declarations will not compile", + ) + self.assertIn( + 'type(cmpfsw_type), pointer :: ptr(:) => null()', text, + "DDT pointer declaration missing type(...) wrapper", + ) + self.assertNotIn( + '\n cmpfsw_type, pointer ::', text, + "Bare DDT name in pointer declaration — invalid Fortran", + ) + + def test_intrinsic_only_emits_no_ddt_use(self): + combos = {('real', 'kind_phys', 1)} + lines = _generate_suite_types('demo', combos, {}) + text = '\n'.join(lines) + self.assertIn('use ccpp_kinds, only: kind_phys', text) + # Sanity: only one USE line total. + self.assertEqual(text.count(' use '), 1) + + def test_distinct_character_lengths_get_distinct_wrappers(self): + """End-to-end: two ``character`` ptr-wrappers of different + lengths must emit two distinct ``type`` declarations. The + ``public ::`` list also carries both names (no duplicates). + Regression for the SCM types-module duplicate-symbol bug.""" + combos = { + ('character', 'len=10', 1), + ('character', 'len=3', 1), + } + lines = _generate_suite_types('demo', combos, {}) + text = '\n'.join(lines) + # Each length appears in exactly one ``type :: ... ptr_type`` block. + self.assertIn( + 'type :: character_len10_rank1_ptr_type', text, + ) + self.assertIn( + 'type :: character_len3_rank1_ptr_type', text, + ) + # And the declarations themselves carry the right Fortran len=. + self.assertIn( + 'character(len=10), pointer :: ptr(:) => null()', text, + ) + self.assertIn( + 'character(len=3), pointer :: ptr(:) => null()', text, + ) + # No duplicate symbol — a name appears exactly twice (``public ::`` + # line and the ``type :: NAME`` opener; ``end type NAME`` does + # not count since ``end`` precedes it). + for name in ('character_len10_rank1_ptr_type', + 'character_len3_rank1_ptr_type'): + self.assertEqual( + text.count('public :: {}'.format(name)), 1, + 'expected one public:: declaration for {}, got: {!r}'.format( + name, + [l for l in lines if name in l], + ), + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/test_suite_xml.py b/unit-tests/test_suite_xml.py new file mode 100644 index 00000000..11d94bac --- /dev/null +++ b/unit-tests/test_suite_xml.py @@ -0,0 +1,769 @@ +#!/usr/bin/env python3 + +"""Unit tests for :mod:`generator.suite_xml`. + +Tests cover: + +1. The in-memory data model: :class:`~generator.suite_xml.SuiteScheme`, + :class:`~generator.suite_xml.SuiteSubcycle`, + :class:`~generator.suite_xml.SuiteSubcol`, + :class:`~generator.suite_xml.SuiteGroup`, + :class:`~generator.suite_xml.Suite`. +2. The XML-to-object builder :func:`~generator.suite_xml._build_suite`. +3. The public API :func:`~generator.suite_xml.parse_suite_xml` — including + nested-suite expansion, expanded-XML writing, and error detection. + +The test cases for parsing and expansion are ported from +``test/unit_tests/test_sdf.py`` in the legacy capgen test suite, updated +for the new package layout. + +Run with:: + + python -m pytest capgen/tests/test_suite_xml.py -v + +or include ``--doctest-modules capgen/generator/suite_xml.py``. +""" + +import filecmp +import glob +import logging +import os +import sys +import tempfile +import unittest +import xml.etree.ElementTree as ET + +# ---- path setup ------------------------------------------------------------ +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_REPO_ROOT = os.path.dirname(_TESTS_DIR) +_PKG_ROOT = os.path.join(_REPO_ROOT, 'capgen') +for _p in (_PKG_ROOT, _REPO_ROOT): + if _p not in sys.path: + sys.path.insert(0, _p) + +# ---- imports ---------------------------------------------------------------- +from metadata.parse_tools import ( + CCPPError, + init_log, + set_log_to_null, + read_xml_file, + find_schema_version, + expand_nested_suites, + write_xml_file, +) +from metadata.parse_tools.xml_tools import validate_xml_file +from generator.suite_xml import ( + Suite, + SuiteGroup, + SuiteScheme, + SuiteSubcol, + SuiteSubcycle, + _build_suite, + _parse_group_items, + parse_suite_xml, +) + +_SAMPLE_DIR = os.path.join(_TESTS_DIR, 'sample_suite_files') +_SCHEMA_DIR = os.path.join(_PKG_ROOT, 'schema') + + +######################################################################## +# Helper — shared logger (quiet by default) +######################################################################## + +def _make_logger(name='test_suite_xml'): + log = init_log(name, level=logging.WARNING) + set_log_to_null(log) + return log + + +######################################################################## +# Helper — XML tree comparison (ported from test_sdf.py) +######################################################################## + +def _compare_text(name, txt1, txt2, typ): + """Return an error string if the text items differ, else None.""" + if txt1 and txt2: + if txt1.strip() != txt2.strip(): + return f"{name} {typ}, '{txt1}', does not match {typ}, '{txt2}'" + elif txt1: + return f"{name} {typ} is missing from string2" + elif txt2: + return f"{name} {typ} is missing from string1" + return None + + +def xml_diff(xt1, xt2): + """Return a list of difference strings between two ElementTree subtrees. + + Returns an empty list when the trees are identical. + """ + diffs = [] + if xt1.tag != xt2.tag: + diffs.append(f"Tags do not match: {xt1.tag} != {xt2.tag}") + return diffs + for name, value in xt1.attrib.items(): + if name not in xt2.attrib: + diffs.append(f"xt1 attribute, {name}, is missing in xt2") + elif xt2.attrib[name] != value: + diffs.append(f"Attributes for {name} do not match: " + f"{value!r} != {xt2.attrib[name]!r}") + for name in xt2.attrib: + if name not in xt1.attrib: + diffs.append(f"xt2 attribute, {name}, is missing in xt1") + tdiff = _compare_text(xt1.tag, xt1.text, xt2.text, "text") + if tdiff: + diffs.append(tdiff) + tdiff = _compare_text(xt1.tag, xt1.tail, xt2.tail, "tail") + if tdiff: + diffs.append(tdiff) + if len(xt1) != len(xt2): + diffs.append(f"Number of children differs: {len(xt1)} != {len(xt2)}") + else: + for c1, c2 in zip(xt1, xt2): + diffs.extend(xml_diff(c1, c2)) + return diffs + + +######################################################################## +# Data model tests +######################################################################## + +class TestSuiteScheme(unittest.TestCase): + """Tests for :class:`SuiteScheme`.""" + + def test_creation(self): + s = SuiteScheme('my_scheme') + self.assertEqual(s.name, 'my_scheme') + + def test_leading_trailing_spaces_stripped(self): + s = SuiteScheme(' spaced ') + self.assertEqual(s.name, 'spaced') + + def test_scheme_names(self): + self.assertEqual(SuiteScheme('foo').scheme_names(), ['foo']) + + def test_repr(self): + self.assertIn('my_scheme', repr(SuiteScheme('my_scheme'))) + + +class TestSuiteSubcycle(unittest.TestCase): + """Tests for :class:`SuiteSubcycle`.""" + + def test_literal_integer_loop(self): + subcycle = SuiteSubcycle(loop='2', items=[]) + self.assertEqual(subcycle.loop, '2') + self.assertTrue(subcycle.is_literal_count) + + def test_stdname_loop(self): + subcycle = SuiteSubcycle(loop='num_subcycles_for_ag', items=[]) + self.assertFalse(subcycle.is_literal_count) + + def test_none_loop_is_literal(self): + subcycle = SuiteSubcycle(loop=None, items=[]) + self.assertIsNone(subcycle.loop) + self.assertTrue(subcycle.is_literal_count) + + def test_scheme_names_from_items(self): + subcycle = SuiteSubcycle(loop='2', items=[ + SuiteScheme('scheme_a'), + SuiteScheme('scheme_b'), + ]) + self.assertEqual(subcycle.scheme_names(), ['scheme_a', 'scheme_b']) + + def test_nested_subcycle_scheme_names(self): + inner = SuiteSubcycle(loop='3', items=[SuiteScheme('inner_sch')]) + outer = SuiteSubcycle(loop='2', items=[SuiteScheme('outer_sch'), inner]) + self.assertEqual(outer.scheme_names(), ['outer_sch', 'inner_sch']) + + +class TestSuiteSubcol(unittest.TestCase): + """Tests for :class:`SuiteSubcol`.""" + + def test_creation(self): + subcycle = SuiteSubcol('gen_routine', 'avg_routine', [SuiteScheme('sch')]) + self.assertEqual(subcycle.gen_routine, 'gen_routine') + self.assertEqual(subcycle.avg_routine, 'avg_routine') + + def test_scheme_names(self): + subcycle = SuiteSubcol('g', 'a', [SuiteScheme('s1'), SuiteScheme('s2')]) + self.assertEqual(subcycle.scheme_names(), ['s1', 's2']) + + +class TestSuiteGroup(unittest.TestCase): + """Tests for :class:`SuiteGroup`.""" + + def test_creation(self): + g = SuiteGroup('physics', [SuiteScheme('sch1'), SuiteScheme('sch2')]) + self.assertEqual(g.name, 'physics') + self.assertEqual(len(g.items), 2) + + def test_scheme_names(self): + g = SuiteGroup('g', [ + SuiteScheme('a'), + SuiteSubcycle('2', [SuiteScheme('b'), SuiteScheme('a')]), + ]) + self.assertEqual(g.scheme_names(), ['a', 'b', 'a']) + + def test_unique_scheme_names(self): + g = SuiteGroup('g', [ + SuiteScheme('a'), + SuiteSubcycle('2', [SuiteScheme('b'), SuiteScheme('a')]), + ]) + self.assertEqual(g.unique_scheme_names(), ['a', 'b']) + + +class TestSuite(unittest.TestCase): + """Tests for :class:`Suite`.""" + + def _make_suite(self, groups=None, init=None, final=None): + groups = groups or [SuiteGroup('g', [SuiteScheme('s')])] + return Suite('my_suite', [2, 0], '/f.xml', groups, init, final) + + def test_name(self): + self.assertEqual(self._make_suite().name, 'my_suite') + + def test_group_names(self): + s = self._make_suite([ + SuiteGroup('g1', [SuiteScheme('s1')]), + SuiteGroup('g2', [SuiteScheme('s2')]), + ]) + self.assertEqual(s.group_names(), ['g1', 'g2']) + + def test_get_group_found(self): + grp = SuiteGroup('dynamics', [SuiteScheme('dyn')]) + s = self._make_suite([grp]) + self.assertIs(s.get_group('dynamics'), grp) + + def test_get_group_not_found(self): + self.assertIsNone(self._make_suite().get_group('nope')) + + def test_all_scheme_names_deduped(self): + s = self._make_suite([ + SuiteGroup('g1', [SuiteScheme('common'), SuiteScheme('a')]), + SuiteGroup('g2', [SuiteScheme('b'), SuiteScheme('common')]), + ]) + self.assertEqual(s.all_scheme_names(), ['common', 'a', 'b']) + + def test_init_final_schemes(self): + s = self._make_suite(init='suite_init', final='suite_final') + self.assertEqual(s.init_scheme, 'suite_init') + self.assertEqual(s.final_scheme, 'suite_final') + + def test_expanded_file_default_none(self): + self.assertIsNone(self._make_suite().expanded_file) + + +######################################################################## +# _parse_group_items tests +######################################################################## + +class TestParseGroupItems(unittest.TestCase): + """Tests for :func:`_parse_group_items`.""" + + def test_single_scheme(self): + xml = 'my_scheme' + el = ET.fromstring(xml) + items = _parse_group_items(el) + self.assertEqual(len(items), 1) + self.assertIsInstance(items[0], SuiteScheme) + self.assertEqual(items[0].name, 'my_scheme') + + def test_subcycle_with_loop(self): + xml = 's' + el = ET.fromstring(xml) + items = _parse_group_items(el) + self.assertEqual(len(items), 1) + subcycle = items[0] + self.assertIsInstance(subcycle, SuiteSubcycle) + self.assertEqual(subcycle.loop, '3') + self.assertEqual(len(subcycle.items), 1) + + def test_subcol(self): + xml = ('' + '' + 's') + el = ET.fromstring(xml) + items = _parse_group_items(el) + self.assertIsInstance(items[0], SuiteSubcol) + self.assertEqual(items[0].gen_routine, 'gen_r') + + def test_subcol_missing_avg_raises(self): + xml = ('' + '' + 's') + el = ET.fromstring(xml) + with self.assertRaises(CCPPError): + _parse_group_items(el) + + def test_empty_scheme_raises(self): + xml = ' ' + el = ET.fromstring(xml) + with self.assertRaises(CCPPError): + _parse_group_items(el) + + def test_nested_subcycles(self): + xml = ''' + + + inner + + + ''' + el = ET.fromstring(xml) + items = _parse_group_items(el) + outer = items[0] + self.assertIsInstance(outer, SuiteSubcycle) + inner = outer.items[0] + self.assertIsInstance(inner, SuiteSubcycle) + self.assertEqual(inner.loop, '3') + self.assertEqual(inner.items[0].name, 'inner') + + +######################################################################## +# _build_suite tests +######################################################################## + +class TestBuildSuite(unittest.TestCase): + """Tests for :func:`_build_suite`.""" + + def _parse(self, xml_str, source='test.xml', version=None): + root = ET.fromstring(xml_str) + v = version or [2, 0] + return _build_suite(root, source, v, _make_logger()) + + def test_basic_suite(self): + xml = ''' + + sch1 + + ''' + suite = self._parse(xml) + self.assertEqual(suite.name, 'my_suite') + self.assertEqual(suite.group_names(), ['physics']) + + def test_suite_with_init_final(self): + xml = ''' + suite_init_scheme + sch + suite_final_scheme + ''' + suite = self._parse(xml) + self.assertEqual(suite.init_scheme, 'suite_init_scheme') + self.assertEqual(suite.final_scheme, 'suite_final_scheme') + + def test_deprecated_finalize_rejected(self): + """ (old long form) is rejected with a clear error + pointing at the canonical short form.""" + xml = ''' + sch + final_scheme + ''' + with self.assertRaises(CCPPError) as cm: + self._parse(xml) + msg = str(cm.exception) + self.assertIn('finalize', msg) + self.assertIn('', msg) + + def test_deprecated_initalize_typo_rejected(self): + """ (the old schema's typo) is rejected with a clear + error pointing at the canonical short form.""" + xml = ''' + init_scheme + sch + ''' + with self.assertRaises(CCPPError) as cm: + self._parse(xml) + msg = str(cm.exception) + self.assertIn('initalize', msg) + self.assertIn('', msg) + + def test_deprecated_initialize_correct_spelling_rejected(self): + """ (the correctly-spelled long form) is also rejected. + Only the short is accepted.""" + xml = ''' + init_scheme + sch + ''' + with self.assertRaises(CCPPError) as cm: + self._parse(xml) + msg = str(cm.exception) + self.assertIn('initialize', msg) + self.assertIn('', msg) + + def test_missing_suite_name_raises(self): + xml = 's' + with self.assertRaises(CCPPError): + self._parse(xml) + + def test_duplicate_group_name_raises(self): + xml = ''' + a + b + ''' + with self.assertRaises(CCPPError) as cm: + self._parse(xml) + self.assertIn('dup', str(cm.exception)) + + def test_empty_init_raises(self): + xml = ''' + + s + ''' + with self.assertRaises(CCPPError): + self._parse(xml) + + def test_empty_final_raises(self): + xml = ''' + s + + ''' + with self.assertRaises(CCPPError): + self._parse(xml) + + def test_subcycle_in_group(self): + xml = ''' + + + scheme6 + + + ''' + suite = self._parse(xml) + grp = suite.groups[0] + subcycle = grp.items[0] + self.assertIsInstance(subcycle, SuiteSubcycle) + self.assertEqual(subcycle.loop, 'num_subcycles_for_scheme6') + self.assertFalse(subcycle.is_literal_count) + + +######################################################################## +# parse_suite_xml — valid files (ported from test_sdf.py) +######################################################################## + +def _sample(name): + return os.path.join(_SAMPLE_DIR, name) + + +class TestParseSuiteXmlValid(unittest.TestCase): + """Integration tests for :func:`parse_suite_xml` with valid SDFs. + + These tests port the ``test_good_v*`` cases from the legacy + ``test_sdf.py`` test suite. Expanded XML output is compared against + the ``*_exp.xml`` reference files. + """ + + def setUp(self): + self._tmp = tempfile.mkdtemp() + self._log = _make_logger() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmp, ignore_errors=True) + + def _parse(self, filename, skip_validation=True): + return parse_suite_xml( + _sample(filename), self._tmp, + logger=self._log, + schema_path=_SCHEMA_DIR, + skip_validation=skip_validation, + ) + + def _compare_expanded(self, suite, exp_filename): + """Assert that the expanded XML matches the reference file.""" + self.assertIsNotNone(suite.expanded_file) + self.assertTrue(os.path.isfile(suite.expanded_file)) + _, ref_root = read_xml_file(_sample(exp_filename), self._log) + _, got_root = read_xml_file(suite.expanded_file, self._log) + diffs = xml_diff(ref_root, got_root) + sep = '\n' + self.assertFalse( + diffs, + msg=f"Expanded XML differs from {exp_filename}:\n{sep.join(diffs)}" + ) + + # ---- v1 suites --------------------------------------------------------- + + def test_v1_suite_01(self): + """V1 SDF is read and written without expansion.""" + suite = self._parse('suite_good_v1_test01.xml') + self.assertEqual(suite.version[0], 1) + self.assertTrue(os.path.isfile(suite.expanded_file)) + + def test_v1_suite_02(self): + suite = self._parse('suite_good_v1_test02.xml') + self.assertEqual(suite.version[0], 1) + + # ---- v2 suites --------------------------------------------------------- + + def test_v2_suite_01_expand_group_of_nested_suite(self): + """Expand one group from a simple nested suite at group level.""" + suite = self._parse('suite_good_v2_test01.xml') + self.assertEqual(suite.name, 'ver_test_suite') + self.assertEqual(suite.version, [2, 0]) + self.assertEqual(suite.group_names(), ['group1']) + self._compare_expanded(suite, 'suite_good_v2_test01_exp.xml') + + def test_v2_suite_01_scheme_names(self): + """After expansion, correct scheme names are in group1.""" + suite = self._parse('suite_good_v2_test01.xml') + names = suite.get_group('group1').scheme_names() + # Expected after expansion: scheme5, scheme1i, scheme2i, scheme1i, scheme9 + self.assertEqual(names, ['scheme5', 'scheme1i', 'scheme2i', 'scheme1i', 'scheme9']) + + def test_v2_suite_02_expand_one_group_of_multigroup_nested_suite(self): + """Expand one group from a multi-group nested suite at group level.""" + suite = self._parse('suite_good_v2_test02.xml') + self.assertEqual(suite.name, 'v2_suite') + self._compare_expanded(suite, 'suite_good_v2_test02_exp.xml') + + def test_v2_suite_02_subcycle_preserved(self): + """Subcycle loop attribute survives the expansion.""" + suite = self._parse('suite_good_v2_test02.xml') + grp = suite.get_group('main_group') + # First item is the subcycle from the original suite + subcycle = grp.items[0] + self.assertIsInstance(subcycle, SuiteSubcycle) + self.assertEqual(subcycle.loop, 'num_subcycles_for_scheme6') + + def test_v2_suite_03_expand_multiple_nested_suites(self): + """Expand two nested suites at group level + full suite at suite level.""" + suite = self._parse('suite_good_v2_test03.xml') + self.assertEqual(suite.name, 'main_suite') + # Should have 3 groups after expansion: groupp, nested_group1, nested_group2 + self.assertEqual(len(suite.groups), 3) + self._compare_expanded(suite, 'suite_good_v2_test03_exp.xml') + + def test_v2_suite_04_expand_group_from_nested_full_suite(self): + """Expand two group-level nested suites + one group from nested suite at suite level.""" + suite = self._parse('suite_good_v2_test04.xml') + self.assertEqual(suite.name, 'ver_test_suite') + # 2 groups: main11 + nested_group2 (only group from nested_full_suite) + self.assertEqual(len(suite.groups), 2) + self._compare_expanded(suite, 'suite_good_v2_test04_exp.xml') + + def test_expanded_xml_written_to_output_root(self): + """The expanded XML must be written to the correct output path.""" + suite = self._parse('suite_good_v2_test01.xml') + expected_name = f"ccpp_{suite.name}_expanded.xml" + expected_path = os.path.join(self._tmp, expected_name) + self.assertEqual(suite.expanded_file, expected_path) + self.assertTrue(os.path.isfile(expected_path)) + + def test_all_scheme_names_unique(self): + """all_scheme_names() deduplicates across groups.""" + suite = self._parse('suite_good_v2_test03.xml') + all_names = suite.all_scheme_names() + self.assertEqual(len(all_names), len(set(all_names))) + + +######################################################################## +# parse_suite_xml — error cases +######################################################################## + +class TestParseSuiteXmlErrors(unittest.TestCase): + """Error handling tests for :func:`parse_suite_xml`.""" + + def setUp(self): + self._tmp = tempfile.mkdtemp() + self._log = _make_logger() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmp, ignore_errors=True) + + def _parse(self, filename, skip_validation=True): + return parse_suite_xml( + _sample(filename), self._tmp, + logger=self._log, + schema_path=_SCHEMA_DIR, + skip_validation=skip_validation, + ) + + def test_nonexistent_file_raises(self): + with self.assertRaises(CCPPError): + parse_suite_xml('/nonexistent/suite.xml', self._tmp, + logger=self._log, skip_validation=True) + + def test_bad_schema_version_formats(self): + """Malformed version attributes are detected on file read.""" + for fname in ('suite_bad_version01.xml', 'suite_bad_version02.xml', + 'suite_bad_version03.xml', 'suite_bad_version04.xml'): + with self.subTest(fname=fname): + with self.assertRaises(CCPPError): + self._parse(fname) + + def test_missing_version_raises(self): + with self.assertRaises(CCPPError) as cm: + self._parse('suite_missing_version.xml') + self.assertIn('suite_missing_version.xml', str(cm.exception)) + self.assertIn('Version attribute required', str(cm.exception)) + + def test_infinite_group_recursion_detected(self): + """Circular nested-suite references at group level are caught.""" + with self.assertRaises(CCPPError) as cm: + self._parse('suite_recurse_top1.xml') + self.assertIn('iterations', str(cm.exception)) + + def test_infinite_suite_recursion_detected(self): + """Circular nested-suite references at suite level are caught.""" + with self.assertRaises(CCPPError) as cm: + self._parse('suite_recurse_top2.xml') + self.assertIn('iterations', str(cm.exception)) + + def test_missing_nested_group_raises(self): + """Referencing a group that doesn't exist in the target suite raises.""" + with self.assertRaises(CCPPError) as cm: + self._parse('suite_missing_group.xml') + self.assertIn('not found', str(cm.exception)) + + def test_missing_loaded_suite_raises(self): + """Referencing a suite name that doesn't exist in the target file raises.""" + with self.assertRaises(CCPPError) as cm: + self._parse('suite_missing_loaded_suite.xml') + self.assertIn('not found', str(cm.exception)) + + +######################################################################## +# Schema validation tests (require xmllint) +######################################################################## + +class TestSchemaValidation(unittest.TestCase): + """Tests that exercise xmllint-based XML schema validation. + + These tests are skipped automatically when ``xmllint`` is not installed. + """ + + @classmethod + def setUpClass(cls): + import shutil + if not shutil.which('xmllint'): + raise unittest.SkipTest("xmllint not installed — skipping schema validation tests") + cls._log = _make_logger() + cls._tmp = tempfile.mkdtemp() + + @classmethod + def tearDownClass(cls): + import shutil + shutil.rmtree(cls._tmp, ignore_errors=True) + + def test_good_v2_suite_validates(self): + _, root = read_xml_file(_sample('suite_good_v2_test01.xml'), self._log) + version = find_schema_version(root) + result = validate_xml_file( + _sample('suite_good_v2_test01.xml'), 'suite', version, self._log, + schema_path=_SCHEMA_DIR + ) + self.assertTrue(result) + + def test_bad_suite_tag_rejected(self): + """A nested element violates the schema.""" + _, root = read_xml_file(_sample('suite_bad_v2_suite_tag.xml'), self._log) + version = find_schema_version(root) + try: + result = validate_xml_file( + _sample('suite_bad_v2_suite_tag.xml'), 'suite', version, + self._log, schema_path=_SCHEMA_DIR + ) + # Some xmllint versions return True even on error + except CCPPError as exc: + self.assertIn("not expected", str(exc)) + + def test_invalid_fortran_id_scheme_rejected(self): + """A scheme name that is not a valid Fortran ID is rejected.""" + _, root = read_xml_file( + _sample('suite_invalid_scheme_fortran_id.xml'), self._log + ) + version = find_schema_version(root) + with self.assertRaises(CCPPError) as cm: + validate_xml_file( + _sample('suite_invalid_scheme_fortran_id.xml'), 'suite', + version, self._log, schema_path=_SCHEMA_DIR + ) + self.assertIn("scheme-1", str(cm.exception)) + + def test_invalid_fortran_id_group_rejected(self): + _, root = read_xml_file( + _sample('suite_invalid_group_fortran_id.xml'), self._log + ) + version = find_schema_version(root) + with self.assertRaises(CCPPError) as cm: + validate_xml_file( + _sample('suite_invalid_group_fortran_id.xml'), 'suite', + version, self._log, schema_path=_SCHEMA_DIR + ) + self.assertIn("group-1", str(cm.exception)) + + def test_invalid_fortran_id_suite_rejected(self): + _, root = read_xml_file( + _sample('suite_invalid_suite_fortran_id.xml'), self._log + ) + version = find_schema_version(root) + with self.assertRaises(CCPPError) as cm: + validate_xml_file( + _sample('suite_invalid_suite_fortran_id.xml'), 'suite', + version, self._log, schema_path=_SCHEMA_DIR + ) + self.assertIn("ver-test-suite", str(cm.exception)) + + def test_duplicate_group_name_rejected_after_expansion(self): + """After nested-suite expansion a duplicate group name fails validation.""" + _, root = read_xml_file(_sample('suite_bad_v2_duplicate_group.xml'), self._log) + version = find_schema_version(root) + # Initial file validates OK + result = validate_xml_file( + _sample('suite_bad_v2_duplicate_group.xml'), 'suite', version, + self._log, schema_path=_SCHEMA_DIR + ) + self.assertTrue(result) + # After expansion the duplicated xs:ID triggers a validation error + expand_nested_suites(root, _SAMPLE_DIR, logger=self._log) + expanded_path = os.path.join(self._tmp, 'dup_group_expanded.xml') + write_xml_file(root, expanded_path, self._log) + with self.assertRaises(CCPPError) as cm: + validate_xml_file(expanded_path, 'suite', version, self._log, + schema_path=_SCHEMA_DIR) + self.assertIn('group1', str(cm.exception)) + + +######################################################################## +# xml_diff helper tests (ported from test_sdf.py) +######################################################################## + +class TestXmlDiff(unittest.TestCase): + """Tests for the :func:`xml_diff` helper.""" + + def test_matching_trees(self): + r1 = ET.fromstring('text') + r2 = ET.fromstring('text') + self.assertEqual(xml_diff(r1, r2), []) + + def test_tag_mismatch(self): + diffs = xml_diff(ET.fromstring('x'), + ET.fromstring('x')) + self.assertEqual(len(diffs), 1) + self.assertIn('Tags', diffs[0]) + + def test_text_mismatch(self): + diffs = xml_diff(ET.fromstring('a'), ET.fromstring('b')) + self.assertEqual(len(diffs), 1) + self.assertIn('does not match', diffs[0]) + + def test_attrib_mismatch(self): + r1 = ET.fromstring('') + r2 = ET.fromstring('') + diffs = xml_diff(r1, r2) + self.assertEqual(len(diffs), 3) + + def test_child_count_mismatch(self): + r1 = ET.fromstring('

') + r2 = ET.fromstring('

') + diffs = xml_diff(r1, r2) + self.assertEqual(len(diffs), 1) + self.assertIn('children', diffs[0]) + + +######################################################################## + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/test_trace.py b/unit-tests/test_trace.py new file mode 100644 index 00000000..7ed78390 --- /dev/null +++ b/unit-tests/test_trace.py @@ -0,0 +1,196 @@ +"""Unit tests for generator.trace.""" + +import doctest +import unittest + +from generator.trace import ( + emit_module_gate, + emit_trace_block, + ensure_error_unit_use, +) + + +class _FakeEntry: + """Minimal HostVarEntry stand-in for the trace helper.""" + + def __init__(self, standard_name, local_name, type_): + self.standard_name = standard_name + self.local_name = local_name + self.type = type_ + + +class TestEmitModuleGate(unittest.TestCase): + + def test_default_off(self): + self.assertEqual( + emit_module_gate(False, ' '), + [' logical, parameter :: trace = .false.'], + ) + + def test_default_on(self): + self.assertEqual( + emit_module_gate(True, ' '), + [' logical, parameter :: trace = .true.'], + ) + + def test_indent_preserved(self): + self.assertEqual( + emit_module_gate(False, ' '), + [' logical, parameter :: trace = .false.'], + ) + + +class TestEnsureErrorUnitUse(unittest.TestCase): + + def test_appends_when_absent(self): + out = ensure_error_unit_use([], ' ') + self.assertEqual( + out, + [' use, intrinsic :: iso_fortran_env, only: error_unit'], + ) + + def test_idempotent(self): + existing = [' use, intrinsic :: iso_fortran_env, only: error_unit'] + out = ensure_error_unit_use(list(existing), ' ') + self.assertEqual(out, existing) + + def test_appends_after_other_uses(self): + out = ensure_error_unit_use([' use foo, only: bar'], ' ') + self.assertEqual(out, [ + ' use foo, only: bar', + ' use, intrinsic :: iso_fortran_env, only: error_unit', + ]) + + def test_no_substring_false_positive(self): + # ``my_error_unit_proxy`` must not be matched as ``error_unit``. + out = ensure_error_unit_use([' use foo, only: my_error_unit_proxy'], ' ') + self.assertEqual(out, [ + ' use foo, only: my_error_unit_proxy', + ' use, intrinsic :: iso_fortran_env, only: error_unit', + ]) + + +class TestEmitTraceBlock(unittest.TestCase): + + def _ctrl_in(self): + return [ + _FakeEntry('horizontal_loop_begin', 'lb', 'integer'), + _FakeEntry('horizontal_loop_end', 'ub', 'integer'), + _FakeEntry('thread_number', 'thread_num', 'integer'), + ] + + def _ctrl_out(self): + return [ + _FakeEntry('ccpp_error_code', 'errflg', 'integer'), + _FakeEntry('ccpp_error_message', 'errmsg', 'character'), + ] + + def test_emits_for_intent_in_dummies(self): + out = emit_trace_block('my_sub', self._ctrl_in(), ' ') + self.assertEqual(out, [ + " if (trace) write(error_unit, " + "'(a,a,1x,i0,a,1x,i0,a,1x,i0)') &", + " 'CCPP TRACE my_sub:', &", + " ' lb=', lb, &", + " ' ub=', ub, &", + " ' thread_num=', thread_num", + ]) + + def test_filters_intent_out(self): + # An intent(out) entry should not appear in the trace. + out = emit_trace_block( + 'my_sub', self._ctrl_in() + self._ctrl_out(), ' ', + ) + joined = '\n'.join(out) + self.assertNotIn('errflg', joined) + self.assertNotIn('errmsg', joined) + self.assertIn('lb', joined) + self.assertIn('thread_num', joined) + + def test_character_wrapped_in_trim(self): + entries = [_FakeEntry('suite_name', 'suite_name', 'character')] + out = emit_trace_block('my_sub', entries, ' ') + self.assertEqual(out, [ + " if (trace) write(error_unit, '(a,a,a)') &", + " 'CCPP TRACE my_sub:', &", + " ' suite_name=', trim(suite_name)", + ]) + + def test_empty_when_only_intent_out(self): + out = emit_trace_block('my_sub', self._ctrl_out(), ' ') + self.assertEqual(out, []) + + def test_empty_when_no_entries(self): + out = emit_trace_block('my_sub', [], ' ') + self.assertEqual(out, []) + + def test_instance_local_appended(self): + out = emit_trace_block( + 'my_sub', self._ctrl_in(), ' ', instance_local='inst_num', + ) + joined = '\n'.join(out) + self.assertIn("' inst_num=', inst_num", joined) + + def test_instance_local_not_duplicated_when_already_in_entries(self): + entries = self._ctrl_in() + [ + _FakeEntry('instance_number', 'inst_num', 'integer'), + ] + out = emit_trace_block( + 'my_sub', entries, ' ', instance_local='inst_num', + ) + joined = '\n'.join(out) + # Should appear exactly once. + self.assertEqual(joined.count(" inst_num=', inst_num"), 1) + + def test_signature_order_preserved(self): + entries = [ + _FakeEntry('a', 'first', 'integer'), + _FakeEntry('b', 'second', 'integer'), + _FakeEntry('c', 'third', 'integer'), + ] + out = emit_trace_block('my_sub', entries, ' ') + first = out.index(" ' first=', first, &") + second = out.index(" ' second=', second, &") + third = out.index(" ' third=', third") + self.assertLess(first, second) + self.assertLess(second, third) + + def test_format_string_mixes_a_and_i0(self): + entries = [ + _FakeEntry('suite_name', 'suite_name', 'character'), + _FakeEntry('horizontal_loop_begin', 'lb', 'integer'), + _FakeEntry('horizontal_loop_end', 'ub', 'integer'), + ] + out = emit_trace_block('my_sub', entries, ' ') + # First line carries the format: ``a`` for trace name, then for + # each item ``a`` (label) followed by ``a`` (char) or + # ``1x,i0`` (integer). + self.assertEqual( + out[0], + " if (trace) write(error_unit, " + "'(a,a,a,a,1x,i0,a,1x,i0)') &", + ) + + def test_continuation_only_after_last_var_omitted(self): + entries = [ + _FakeEntry('a', 'one', 'integer'), + _FakeEntry('b', 'two', 'integer'), + ] + out = emit_trace_block('my_sub', entries, ' ') + # Every line but the last must end with ``&``. + for line in out[:-1]: + self.assertTrue( + line.rstrip().endswith('&'), + msg='line missing continuation: {!r}'.format(line), + ) + self.assertFalse(out[-1].rstrip().endswith('&')) + + +def load_tests(loader, tests, ignore): + import generator.trace as t + tests.addTests(doctest.DocTestSuite(t)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_validator.py b/unit-tests/test_validator.py new file mode 100644 index 00000000..eb05b49e --- /dev/null +++ b/unit-tests/test_validator.py @@ -0,0 +1,2070 @@ +"""Unit tests for ccpp_validator.""" + +import doctest +import logging +import os +import tempfile +import textwrap +import unittest + +import ccpp_validator as val_mod +from ccpp_validator import ( + _base_local_name, + _join_continuation, + _load_modules_tree, + _load_source_tree, + _parse_modules, + _parse_subroutines, + validate, +) +from metadata.parse_tools import CCPPError + +_SAMPLE_DIR = os.path.join(os.path.dirname(__file__), 'sample_files') +_CORRECT_F90 = os.path.join(_SAMPLE_DIR, 'scheme_multipart_correct.F90') +_WRONG_F90 = os.path.join(_SAMPLE_DIR, 'scheme_multipart_wrong_args.F90') +_SCHEME_META = os.path.join(_SAMPLE_DIR, 'scheme_multipart.meta') + + +class TestJoinContinuation(unittest.TestCase): + + def test_no_continuation(self): + lines = [' foo\n', ' bar\n'] + self.assertEqual(_join_continuation(lines), [' foo', ' bar']) + + def test_single_continuation(self): + lines = [' foo &\n', ' bar\n'] + result = _join_continuation(lines) + self.assertEqual(len(result), 1) + self.assertIn('foo', result[0]) + self.assertIn('bar', result[0]) + + def test_multi_continuation(self): + lines = [' a &\n', ' b &\n', ' c\n'] + result = _join_continuation(lines) + self.assertEqual(len(result), 1) + self.assertIn('a', result[0]) + self.assertIn('c', result[0]) + + def test_comment_after_continuation(self): + lines = [' foo & ! comment\n', ' bar\n'] + result = _join_continuation(lines) + self.assertEqual(len(result), 1) + self.assertNotIn('comment', result[0]) + + def test_dual_form_strips_leading_ampersand(self): + # F77 / fixed-form: ``&`` at column 6 of the continuation line + # is a continuation marker and must be stripped before the + # continued expression is appended to the buffer. Without + # this, the joined logical line carries ``&`` glued into the + # middle of the expression. + lines = [' foo &\n', ' & bar &\n', ' & baz\n'] + result = _join_continuation(lines) + self.assertEqual(len(result), 1) + self.assertNotIn('&', result[0]) + self.assertIn('foo', result[0]) + self.assertIn('bar', result[0]) + self.assertIn('baz', result[0]) + + def test_dual_form_then_free_form_mixed(self): + # Files that use dual-form for one signature and free-form for + # another (or mix within a single signature) must still join + # correctly. + lines = [ + ' start &\n', # trailing & + ' & middle &\n', # leading & + trailing & + ' end\n', # no leading &, plain continuation tail + ] + result = _join_continuation(lines) + self.assertEqual(len(result), 1) + self.assertNotIn('&', result[0]) + for tok in ('start', 'middle', 'end'): + self.assertIn(tok, result[0]) + + def test_comment_line_between_continuation_lines(self): + # Real-world case (sfc_diff.f::stability): a comment-only line + # appears between two continuation lines. Fortran 90+ allows + # this; the join must skip the comment line, not terminate. + lines = [ + ' subroutine stability &\n', + '! --- inputs:\n', + ' & ( z1, zvfun, grav, &\n', + '! --- outputs:\n', + ' & rb, fm, fh, cm, ch, ustar)\n', + ] + result = _join_continuation(lines) + self.assertEqual(len(result), 1) + self.assertIn('subroutine stability', result[0]) + self.assertIn('z1', result[0]) + self.assertIn('ustar', result[0]) + # Closing paren survives. + self.assertIn(')', result[0]) + self.assertNotIn('&', result[0]) + + def test_blank_line_between_continuation_lines(self): + # Blank lines mid-continuation are also legal under F90+. + lines = [ + ' foo &\n', + '\n', + ' & bar &\n', + ' \n', + ' & baz\n', + ] + result = _join_continuation(lines) + self.assertEqual(len(result), 1) + for tok in ('foo', 'bar', 'baz'): + self.assertIn(tok, result[0]) + + def test_sfc_sice_style_missing_trailing_ampersand(self): + """Fixed-form continuation where the second-to-last line has NO + trailing ``&`` but the next line has a column-6 ``&`` is a + valid F77 continuation. CCPP physics has this in the wild + (``SFC_Models/SeaIce/CICE/sfc_sice.f::sfc_sice_run``). Without + look-ahead at the next line's column-6 marker the parser ends + the logical line one step early and the closing ``)`` lands on + its own — args list never closes, signature regex captures 0 + args, validator reports a bogus arg-count mismatch.""" + src_lines = [ + ' subroutine sfc_sice_run &\n', + ' & ( im, kice, ps, t1, &\n', + ' & errmsg, errflg\n', # NO trailing ``&`` + ' & )\n', # column-6 ``&`` only + ] + result = _join_continuation(src_lines) + self.assertEqual(len(result), 1) + # Closing ``)`` must be present in the joined line. + self.assertIn(')', result[0]) + # And no stray leading ``&`` remained in the joined output. + import re as _re + self.assertIsNotNone( + _re.search(r'subroutine\s+sfc_sice_run\s*\([^)]*\)', result[0]), + 'joined signature lacks ``subroutine NAME (args)`` shape — got: {!r}' + .format(result[0]), + ) + + def test_rrtmg_style_signature_round_trip(self): + # The real-world failure mode that motivated the fix: a + # fixed-form subroutine signature with 57 args spread across + # 16 dual-form continuation lines. After joining, ``(`` must + # appear immediately after the subroutine name (no stray + # ``&`` in between) so the signature regex can pick up the + # arg list. + src_lines = [ + ' subroutine rrtmg_lw_run &\n', + ' & ( plyr,plvl,tlyr,tlvl,qlyr,olyr, &\n', + ' & icseed,aeraod,aerssa, &\n', + ' & errmsg, errflg &\n', + ' & )\n', + ] + result = _join_continuation(src_lines) + self.assertEqual(len(result), 1) + self.assertNotIn('&', result[0]) + # ``subroutine rrtmg_lw_run`` is followed (after whitespace) by ``(``. + import re as _re + self.assertIsNotNone( + _re.search(r'subroutine\s+rrtmg_lw_run\s*\(', result[0]), + 'joined signature lacks ``subroutine NAME (`` shape — got: {!r}' + .format(result[0]), + ) + + def test_sfc_ocean_style_decorated_trailing_ampersand(self): + """Fixed-form continuation where the trailing ``&`` is followed by + a stray ``,`` and an inline comment. In strict F77, columns + past 72 are ignored, so ``&, ! --- inputs`` past col-71 is + invisible to the compiler. The parser must not glue the ``,`` + into the joined arg list — otherwise a phantom ``&`` token + appears between args. Triggers the decoration-repair branch + (next line's column-6 ``&`` confirms continuation).""" + src_lines = [ + ' subroutine sfc_ocean_run &\n', + ' & ( im, hvap, cp, &\n', + ' & wind, &, ! --- inputs\n', + ' & errmsg, errflg )\n', + ] + result = _join_continuation(src_lines, filename='sfc_ocean.F') + self.assertEqual(len(result), 1) + # Phantom ``&`` must not appear in the joined logical line. + self.assertNotIn('&', result[0]) + # All real args must still be present. + for tok in ('im', 'hvap', 'cp', 'wind', 'errmsg', 'errflg'): + self.assertIn(tok, result[0]) + # The stray comma from the decoration must NOT survive past + # ``wind`` (no double-comma). + self.assertNotIn(',,', result[0].replace(' ', '')) + + def test_decoration_repair_preserves_real_tokens_past_amp(self): + """When tokens past ``&`` look like real identifiers, the line + is returned untouched so the parser surfaces a real error + instead of silently dropping code.""" + # A pathological line: ``&`` mid-line with an identifier after. + # Next line is col-6 ``&`` so we enter the look-ahead branch. + src_lines = [ + ' foo = a & extra_token\n', + ' & + b\n', + ] + result = _join_continuation(src_lines, filename='/tmp/path.f') + # The line is left untouched (no repair); ``extra_token`` stays. + self.assertEqual(len(result), 1) + self.assertIn('extra_token', result[0]) + + def test_decoration_repair_emits_warning(self): + """The decoration-repair branch must emit a single + ``logger.warning`` naming the file:line so users see that their + source has decoration past the statement end.""" + import logging as _logging + src_lines = [ + ' subroutine foo( &\n', + ' & a, b, &, ! decoration\n', + ' & c )\n', + ' end subroutine foo\n', + ] + # Capture warnings from the validator module's logger. + records = [] + + class _Capture(_logging.Handler): + def emit(self, record): + records.append(record) + + cap = _Capture(level=_logging.WARNING) + val_mod._LOGGER.addHandler(cap) + try: + _join_continuation(src_lines, filename='/some/path/foo.F') + finally: + val_mod._LOGGER.removeHandler(cap) + self.assertEqual(len(records), 1, "expected exactly one warning") + msg = records[0].getMessage() + self.assertIn('/some/path/foo.F', msg) + self.assertIn(':2:', msg) # decoration is on line 2 of src_lines + + +class TestParseSubroutines(unittest.TestCase): + + def test_simple_subroutine(self): + src = 'subroutine foo(a, b, c)\nend subroutine foo\n' + result = _parse_subroutines(src) + self.assertIn('foo', result) + self.assertEqual(result['foo'].args, ['a', 'b', 'c']) + self.assertEqual(result['foo'].optional, set()) + + def test_case_insensitive_name(self): + src = 'SUBROUTINE MyScheme_run(errmsg, errflg)\nend subroutine\n' + result = _parse_subroutines(src) + self.assertIn('myscheme_run', result) + + def test_no_args(self): + src = 'subroutine bar()\nend subroutine bar\n' + result = _parse_subroutines(src) + self.assertEqual(result['bar'].args, []) + + def test_pure_prefix(self): + src = 'pure subroutine baz(x)\nend subroutine\n' + result = _parse_subroutines(src) + self.assertIn('baz', result) + self.assertEqual(result['baz'].args, ['x']) + + def test_elemental_prefix(self): + src = 'elemental subroutine qux(y)\nend subroutine\n' + result = _parse_subroutines(src) + self.assertIn('qux', result) + + def test_continuation_args(self): + src = 'subroutine foo(a, b, &\n c, d)\nend subroutine\n' + result = _parse_subroutines(src) + self.assertIn('foo', result) + self.assertEqual(sorted(result['foo'].args), ['a', 'b', 'c', 'd']) + + def test_multiple_subroutines(self): + src = ( + 'subroutine foo(a)\nend subroutine foo\n' + 'subroutine bar(x, y)\nend subroutine bar\n' + ) + result = _parse_subroutines(src) + self.assertIn('foo', result) + self.assertIn('bar', result) + + def test_nested_subroutine_ignored_if_same_name(self): + # first occurrence wins + src = ( + 'subroutine foo(a)\n' + ' subroutine foo(b, c)\n' + ' end subroutine\n' + 'end subroutine foo\n' + ) + result = _parse_subroutines(src) + self.assertEqual(result['foo'].args, ['a']) + + def test_no_parentheses(self): + # some Fortran compilers allow omitting () for no-arg subs + src = 'subroutine foo\nend subroutine foo\n' + result = _parse_subroutines(src) + self.assertIn('foo', result) + self.assertEqual(result['foo'].args, []) + + def test_fixed_form_dual_continuation_signature(self): + # Regression: rrtmg_lw_run-style F77 signatures use ``&`` at + # both line ends. Pre-fix this gave ``args == []``. + src = ( + ' subroutine rrtmg_lw_run &\n' + ' & ( plyr,plvl,tlyr,tlvl,qlyr,olyr, &\n' + ' & icseed,aeraod,aerssa, sfemis, sfgtmp, &\n' + ' & errmsg, errflg &\n' + ' & )\n' + ' end subroutine rrtmg_lw_run\n' + ) + result = _parse_subroutines(src) + self.assertIn('rrtmg_lw_run', result) + args = result['rrtmg_lw_run'].args + self.assertEqual(args, [ + 'plyr', 'plvl', 'tlyr', 'tlvl', 'qlyr', 'olyr', + 'icseed', 'aeraod', 'aerssa', 'sfemis', 'sfgtmp', + 'errmsg', 'errflg', + ]) + + +class TestLoadSourceTree(unittest.TestCase): + + def test_loads_correct_file(self): + tree = _load_source_tree([_CORRECT_F90]) + self.assertIn('temp_calc_adjust_run', tree) + self.assertIn('temp_calc_adjust_init', tree) + self.assertIn('temp_calc_adjust_final', tree) + + def test_args_from_correct_file(self): + tree = _load_source_tree([_CORRECT_F90]) + run_args = tree['temp_calc_adjust_run'].args + self.assertEqual(sorted(run_args), ['errflg', 'errmsg', 'im', 'temp', 'timestep']) + + def test_merges_multiple_files(self): + src1 = textwrap.dedent("""\ + subroutine aaa(x) + end subroutine aaa + """) + src2 = textwrap.dedent("""\ + subroutine bbb(y, z) + end subroutine bbb + """) + with tempfile.NamedTemporaryFile(suffix='.F90', mode='w', delete=False) as f1: + f1.write(src1) + p1 = f1.name + with tempfile.NamedTemporaryFile(suffix='.F90', mode='w', delete=False) as f2: + f2.write(src2) + p2 = f2.name + try: + tree = _load_source_tree([p1, p2]) + self.assertIn('aaa', tree) + self.assertIn('bbb', tree) + finally: + os.unlink(p1) + os.unlink(p2) + + +class TestValidateCorrect(unittest.TestCase): + + def test_no_errors_on_correct_source(self): + errors = validate([_SCHEME_META], [_CORRECT_F90]) + self.assertEqual(errors, []) + + +class TestValidateWrongArgs(unittest.TestCase): + + def setUp(self): + self.errors = validate([_SCHEME_META], [_WRONG_F90]) + + def test_has_errors(self): + self.assertGreater(len(self.errors), 0) + + def test_arg_count_error_for_init(self): + init_errs = [e for e in self.errors if 'temp_calc_adjust_init' in e] + self.assertGreater(len(init_errs), 0) + + def test_renamed_arg_error_for_run(self): + run_errs = [e for e in self.errors if 'temp_calc_adjust_run' in e] + self.assertGreater(len(run_errs), 0) + + +class TestDegenerateParseHint(unittest.TestCase): + """When the Fortran signature parser finds a subroutine but extracts + zero args while metadata declares many, the error message must + surface a HINT pointing at the parser rather than masquerading as + a routine mismatch. Triggered most commonly by an unsupported + continuation style; reproduced here with a parens-less Fortran + sub paired with multi-arg metadata.""" + + _META = ( + '[ccpp-table-properties]\n' + ' name = bogus_scheme\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = bogus_scheme_run\n' + ' type = scheme\n' + '[ a ]\n' + ' standard_name = horizontal_dimension\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + ' intent = in\n' + '[ b ]\n' + ' standard_name = air_temperature\n' + ' units = K\n' + ' dimensions = (horizontal_dimension)\n' + ' type = real | kind = kind_phys\n' + ' intent = inout\n' + ) + + _F90 = ( + '! Subroutine has no parentheses → parser yields zero args, but\n' + '! the metadata declares two. Hint must fire.\n' + 'subroutine bogus_scheme_run\n' + 'end subroutine bogus_scheme_run\n' + ) + + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + self.meta_path = os.path.join(self.tmpdir, 'bogus_scheme.meta') + self.f90_path = os.path.join(self.tmpdir, 'bogus_scheme.F90') + with open(self.meta_path, 'w') as fh: + fh.write(self._META) + with open(self.f90_path, 'w') as fh: + fh.write(self._F90) + + def tearDown(self): + import shutil + shutil.rmtree(self.tmpdir, ignore_errors=True) + + def test_hint_fires_on_zero_fortran_args(self): + errors = validate([self.meta_path], [self.f90_path]) + count_errs = [e for e in errors if 'Argument count mismatch' in e] + self.assertEqual(len(count_errs), 1, errors) + msg = count_errs[0] + self.assertIn('HINT', msg) + self.assertIn('zero arguments', msg) + self.assertIn('parser', msg) + + +class TestOptionalArgsParsing(unittest.TestCase): + """_parse_subroutines collects optional-attribute arg names.""" + + def test_optional_first_attr(self): + src = textwrap.dedent("""\ + subroutine foo(a, b) + integer, optional, intent(in) :: a + integer, intent(in) :: b + end subroutine foo + """) + sig = _parse_subroutines(src)['foo'] + self.assertEqual(sig.args, ['a', 'b']) + self.assertEqual(sig.optional, {'a'}) + + def test_optional_after_intent(self): + src = textwrap.dedent("""\ + subroutine foo(a, b) + integer, intent(out), optional :: b + integer, intent(in) :: a + end subroutine foo + """) + sig = _parse_subroutines(src)['foo'] + self.assertEqual(sig.optional, {'b'}) + + def test_multiple_vars_one_decl(self): + src = textwrap.dedent("""\ + subroutine foo(a, b, c) + real, optional :: a, b(:,:), c + end subroutine foo + """) + sig = _parse_subroutines(src)['foo'] + self.assertEqual(sig.optional, {'a', 'b', 'c'}) + + def test_no_optional(self): + src = textwrap.dedent("""\ + subroutine foo(a) + integer, intent(in) :: a + end subroutine foo + """) + sig = _parse_subroutines(src)['foo'] + self.assertEqual(sig.optional, set()) + + def test_optional_token_in_string_or_comment_ignored(self): + src = textwrap.dedent("""\ + subroutine foo(a) + integer, intent(in) :: a ! this is optional, but a comment + end subroutine foo + """) + sig = _parse_subroutines(src)['foo'] + self.assertEqual(sig.optional, set()) + + +class TestValidateOptionalArgs(unittest.TestCase): + """Optional Fortran-only args are silently allowed in validation.""" + + _META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ a ] + standard_name = std_a + units = none + dimensions = () + type = integer + intent = in + [ b ] + standard_name = std_b + units = none + dimensions = () + type = integer + intent = in + """) + + _F90_OK = textwrap.dedent("""\ + module my_scheme + contains + subroutine my_scheme_run(a, b, c, d, e) + integer, intent(in) :: a + integer, intent(in) :: b + integer, optional, intent(in) :: c + integer, optional, intent(out) :: d + integer, optional, intent(in) :: e + end subroutine my_scheme_run + end module my_scheme + """) + + _F90_REQUIRED_EXTRA = textwrap.dedent("""\ + module my_scheme + contains + subroutine my_scheme_run(a, b, c) + integer, intent(in) :: a + integer, intent(in) :: b + integer, intent(in) :: c + end subroutine my_scheme_run + end module my_scheme + """) + + def _write_files(self, fortran_src): + with tempfile.NamedTemporaryFile(suffix='.meta', mode='w', delete=False) as fm: + fm.write(self._META) + meta_path = fm.name + with tempfile.NamedTemporaryFile(suffix='.F90', mode='w', delete=False) as ff: + ff.write(fortran_src) + f90_path = ff.name + self._cleanup = [meta_path, f90_path] + return meta_path, f90_path + + def tearDown(self): + for f in getattr(self, '_cleanup', []): + os.unlink(f) + + def test_optional_fortran_only_args_allowed(self): + meta, f90 = self._write_files(self._F90_OK) + errors = validate([meta], [f90]) + self.assertEqual(errors, [], 'unexpected errors: ' + repr(errors)) + + def test_non_optional_extra_fortran_arg_errors(self): + meta, f90 = self._write_files(self._F90_REQUIRED_EXTRA) + errors = validate([meta], [f90]) + self.assertTrue(any('Non-optional arguments in Fortran' in e + for e in errors), + 'expected non-optional-extra error, got: ' + repr(errors)) + + +class TestValidateMissingSubroutine(unittest.TestCase): + + def setUp(self): + # Only supply a file with temp_calc_adjust_init, missing run and final. + src = textwrap.dedent("""\ + module temp_calc_adjust + contains + subroutine temp_calc_adjust_init(im, errmsg, errflg) + end subroutine temp_calc_adjust_init + end module temp_calc_adjust + """) + with tempfile.NamedTemporaryFile(suffix='.F90', mode='w', delete=False) as fh: + fh.write(src) + self._f90 = fh.name + + def tearDown(self): + os.unlink(self._f90) + + def test_missing_subroutine_reported(self): + errors = validate([_SCHEME_META], [self._f90]) + missing = [e for e in errors if 'not found' in e] + self.assertGreater(len(missing), 0) + + +class TestValidateMultipleSources(unittest.TestCase): + """Subroutines split across multiple files.""" + + def setUp(self): + init_src = textwrap.dedent("""\ + module a + contains + subroutine temp_calc_adjust_init(im, errmsg, errflg) + end subroutine + end module a + """) + run_src = textwrap.dedent("""\ + module b + contains + subroutine temp_calc_adjust_run(im, timestep, temp, errmsg, errflg) + end subroutine + end module b + """) + final_src = textwrap.dedent("""\ + module c + contains + subroutine temp_calc_adjust_final(errmsg, errflg) + end subroutine + end module c + """) + self._files = [] + for src in (init_src, run_src, final_src): + with tempfile.NamedTemporaryFile(suffix='.F90', mode='w', delete=False) as fh: + fh.write(src) + self._files.append(fh.name) + + def tearDown(self): + for f in self._files: + os.unlink(f) + + def test_no_errors_split_across_files(self): + errors = validate([_SCHEME_META], self._files) + self.assertEqual(errors, []) + + +class TestSourcePathAutoDiscovery(unittest.TestCase): + """Validator auto-discovers .F90 when source_files is omitted.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + # Write a .meta file whose source_path points to a subdirectory. + self._src_subdir = os.path.join(self._tmpdir, 'fortran') + os.makedirs(self._src_subdir) + + meta_src = textwrap.dedent("""\ + [ccpp-table-properties] + name = myscheme + type = scheme + source_path = fortran + [ccpp-arg-table] + name = myscheme_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + [ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out + """) + self._meta = os.path.join(self._tmpdir, 'myscheme.meta') + with open(self._meta, 'w') as fh: + fh.write(meta_src) + + # Correct .F90 in the source_path subdirectory. + fort_correct = textwrap.dedent("""\ + module myscheme + contains + subroutine myscheme_run(errmsg, errflg) + end subroutine myscheme_run + end module myscheme + """) + self._fort = os.path.join(self._src_subdir, 'myscheme.F90') + with open(self._fort, 'w') as fh: + fh.write(fort_correct) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_auto_discovers_correct_source(self): + errors = validate([self._meta]) + self.assertEqual(errors, []) + + def test_auto_discovers_wrong_source(self): + # Overwrite with wrong arg list. + with open(self._fort, 'w') as fh: + fh.write(textwrap.dedent("""\ + module myscheme + contains + subroutine myscheme_run(errmsg) + end subroutine myscheme_run + end module myscheme + """)) + errors = validate([self._meta]) + self.assertGreater(len(errors), 0) + + +class TestFortranFileForTable(unittest.TestCase): + """Tests for _fortran_file_for_table helper.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def _make_table(self, source_path=''): + from metadata.parse_tools import ParseContext + from metadata.metadata_table import MetadataTable + meta = os.path.join(self._tmpdir, 'foo.meta') + open(meta, 'w').close() + ctx = ParseContext(0, meta) + t = MetadataTable('foo', 'scheme', meta, ctx) + props = {} + if source_path: + props['source_path'] = source_path + t.apply_table_props(props) + return t + + def test_finds_F90_in_meta_dir(self): + fort = os.path.join(self._tmpdir, 'foo.F90') + open(fort, 'w').close() + t = self._make_table() + result = val_mod._fortran_file_for_table(t) + self.assertEqual(result, fort) + + def test_finds_F90_in_source_path(self): + subdir = os.path.join(self._tmpdir, 'src') + os.makedirs(subdir) + fort = os.path.join(subdir, 'foo.F90') + open(fort, 'w').close() + t = self._make_table(source_path='src') + result = val_mod._fortran_file_for_table(t) + self.assertEqual(result, fort) + + def test_returns_none_when_not_found(self): + t = self._make_table() + result = val_mod._fortran_file_for_table(t) + self.assertIsNone(result) + + +class TestArgAttributeChecks(unittest.TestCase): + """Per-arg type/kind/intent/rank/optional mismatch detection. + + Each test builds a tiny in-memory metadata + Fortran source pair and + runs ``validate`` end-to-end. The Fortran source has the same arg + names as the metadata so the name-set check passes; we deliberately + perturb one attribute per test to exercise one check at a time. + """ + + _BASE_META = ( + '[ccpp-table-properties]\n' + ' name = s\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = s_run\n' + ' type = scheme\n' + '[ a ]\n' + ' standard_name = a_std\n' + ' units = 1\n' + ' dimensions = ()\n' + ' type = integer\n' + ' intent = {intent_a}\n' + '[ b ]\n' + ' standard_name = b_std\n' + ' units = K\n' + ' dimensions = {dims_b}\n' + ' type = {type_b}\n' + ' kind = {kind_b}\n' + ' intent = {intent_b}\n' + ' optional = {optional_b}\n' + ) + + def _run(self, meta_text, f90_text): + with tempfile.TemporaryDirectory() as d: + meta_path = os.path.join(d, 's.meta') + f90_path = os.path.join(d, 's.F90') + with open(meta_path, 'w') as fh: + fh.write(meta_text) + with open(f90_path, 'w') as fh: + fh.write(f90_text) + return validate([meta_path], [f90_path]) + + def _meta(self, **overrides): + defaults = dict(intent_a='in', dims_b='()', type_b='real', + kind_b='kind_phys', intent_b='in', optional_b='False') + defaults.update(overrides) + return self._BASE_META.format(**defaults) + + _F90_TEMPLATE = ( + 'module m\n' + 'contains\n' + ' subroutine s_run({sig})\n' + ' use ccpp_kinds, only: kind_phys\n' + '{decls}' + ' end subroutine s_run\n' + 'end module m\n' + ) + + def _f90(self, sig, decls): + return self._F90_TEMPLATE.format(sig=sig, decls=decls) + + def test_clean_match_no_errors(self): + errs = self._run( + self._meta(), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' real(kind=kind_phys), intent(in) :: b\n' + )), + ) + self.assertEqual(errs, []) + + def test_intent_mismatch(self): + errs = self._run( + self._meta(intent_b='in'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' real(kind=kind_phys), intent(out) :: b\n' + )), + ) + self.assertTrue(any("intent mismatch" in e and "'b'" in e for e in errs), + msg=errs) + + def test_type_mismatch(self): + errs = self._run( + self._meta(type_b='real'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' integer, intent(in) :: b\n' + )), + ) + self.assertTrue(any("type mismatch" in e and "'b'" in e for e in errs), + msg=errs) + + def test_kind_mismatch(self): + errs = self._run( + self._meta(kind_b='kind_phys'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' real, intent(in) :: b\n' # missing kind + )), + ) + self.assertTrue(any("kind mismatch" in e and "'b'" in e for e in errs), + msg=errs) + + def test_character_len_star_consistent_passes(self): + # len=* on BOTH sides is consistent -> no error. + errs = self._run( + self._meta(type_b='character', kind_b='len=*'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' character(len=*), intent(in) :: b\n' + )), + ) + char_errs = [e for e in errs if "'b'" in e and 'character' in e] + self.assertEqual(char_errs, [], msg=errs) + + def test_character_len_star_vs_concrete_is_mismatch(self): + # len=* must NOT wildcard against a concrete len=N: the metadata must + # mirror the Fortran exactly. + errs = self._run( + self._meta(type_b='character', kind_b='len=512'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' character(len=*), intent(in) :: b\n' + )), + ) + self.assertTrue( + any("character length mismatch" in e and "'b'" in e for e in errs), + msg=errs, + ) + + def test_character_concrete_len_match_passes(self): + # Identical concrete lengths agree. + errs = self._run( + self._meta(type_b='character', kind_b='len=64'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' character(len=64), intent(in) :: b\n' + )), + ) + char_errs = [e for e in errs if "'b'" in e and 'character' in e] + self.assertEqual(char_errs, [], msg=errs) + + def test_character_old_style_len_parsed_and_matched(self): + # Old-style F77 character*64 in Fortran is normalised to len=64 and + # matched against the metadata. + errs = self._run( + self._meta(type_b='character', kind_b='len=64'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' character*64, intent(in) :: b\n' + )), + ) + char_errs = [e for e in errs if "'b'" in e and 'character' in e] + self.assertEqual(char_errs, [], msg=errs) + + def test_rank_mismatch(self): + errs = self._run( + self._meta(dims_b='(d1, d2)'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' real(kind=kind_phys), intent(in) :: b\n' # rank 0, metadata says rank 2 + )), + ) + self.assertTrue(any("rank mismatch" in e and "'b'" in e for e in errs), + msg=errs) + + def test_rank_via_var_attached_dims(self): + errs = self._run( + self._meta(dims_b='(d1, d2)'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' real(kind=kind_phys), intent(in) :: b(:,:)\n' + )), + ) + self.assertEqual(errs, []) + + def test_metadata_optional_but_fortran_required_is_error(self): + # Metadata says optional=True, Fortran doesn't carry the + # 'optional' attribute -> hard error (cap would emit invalid + # present() checks on a required dummy). + errs = self._run( + self._meta(optional_b='True'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' real(kind=kind_phys), intent(in) :: b\n' + )), + ) + self.assertTrue( + any("optional=True" in e and "'b'" in e for e in errs), + msg=errs, + ) + + def test_ddt_metadata_bare_name_matches_fortran_type_wrapper(self): + # Metadata: type = ty_rad_lw (bare DDT name). + # Fortran: type(ty_rad_lw), intent(in) :: b + # These should compare equal after type-name normalisation. + errs = self._run( + self._meta(type_b='ty_rad_lw', kind_b=''), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' type(ty_rad_lw), intent(in) :: b\n' + )), + ) + b_errs = [e for e in errs if "'b'" in e] + self.assertEqual(b_errs, [], msg=errs) + + def test_ddt_class_wrapper_matches_metadata_bare_name(self): + # Fortran polymorphic wrapper: class(...) on the Fortran side + # still matches a bare DDT name in metadata. + errs = self._run( + self._meta(type_b='ty_rad_lw', kind_b=''), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' class(ty_rad_lw), intent(in) :: b\n' + )), + ) + b_errs = [e for e in errs if "'b'" in e] + self.assertEqual(b_errs, [], msg=errs) + + def test_ddt_name_mismatch_is_error(self): + # Different DDT names on each side -> error. + errs = self._run( + self._meta(type_b='ty_rad_lw', kind_b=''), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' type(ty_rad_sw), intent(in) :: b\n' + )), + ) + self.assertTrue( + any("type mismatch" in e and "'b'" in e for e in errs), + msg=errs, + ) + + def test_external_type_matches_fortran_bare_typename(self): + # Metadata: type = external:mpi_f08:mpi_comm (module + name). + # Fortran: type(mpi_comm), intent(in) :: b + # The module is metadata-only; Fortran sees the bare typename. + errs = self._run( + self._meta(type_b='external:mpi_f08:mpi_comm', kind_b=''), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' type(mpi_comm), intent(in) :: b\n' + )), + ) + b_errs = [e for e in errs if "'b'" in e] + self.assertEqual(b_errs, [], msg=errs) + + def test_external_type_mismatched_typename_is_error(self): + errs = self._run( + self._meta(type_b='external:mpi_f08:mpi_comm', kind_b=''), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' type(mpi_request), intent(in) :: b\n' + )), + ) + self.assertTrue( + any("type mismatch" in e and "'b'" in e for e in errs), + msg=errs, + ) + + def test_fortran_optional_but_metadata_required_is_warning(self): + # Reverse direction: metadata=False (default), Fortran=optional + # -> NOT an error. The cap always passes the arg; that's a + # valid subset of the Fortran contract. A warning is emitted. + import io + log_buf = io.StringIO() + handler = logging.StreamHandler(log_buf) + handler.setLevel(logging.WARNING) + log = logging.getLogger('test_validator_fopt_metareq') + log.addHandler(handler) + log.setLevel(logging.WARNING) + try: + with tempfile.TemporaryDirectory() as d: + meta_path = os.path.join(d, 's.meta') + f90_path = os.path.join(d, 's.F90') + with open(meta_path, 'w') as fh: + fh.write(self._meta()) + with open(f90_path, 'w') as fh: + fh.write(self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' real(kind=kind_phys), optional, intent(in) :: b\n' + ))) + errs = validate([meta_path], [f90_path], logger=log) + finally: + log.removeHandler(handler) + # No errors. + b_errs = [e for e in errs if "'b'" in e] + self.assertEqual(b_errs, [], msg=errs) + # But a warning for 'b'. + self.assertIn("Fortran argument 'b'", log_buf.getvalue()) + self.assertIn("optional", log_buf.getvalue()) + + +class TestFortranOnlyOptionalWarning(unittest.TestCase): + """A Fortran-optional arg absent from metadata triggers a logger.warning + but no validation error.""" + + _META = ( + '[ccpp-table-properties]\n' + ' name = s\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = s_run\n' + ' type = scheme\n' + '[ a ]\n' + ' standard_name = a_std\n' + ' units = 1\n' + ' dimensions = ()\n' + ' type = integer\n' + ' intent = in\n' + ) + + _F90 = ( + 'module m\n' + 'contains\n' + ' subroutine s_run(a, b)\n' + ' integer, intent(in) :: a\n' + ' integer, optional, intent(in) :: b\n' + ' end subroutine s_run\n' + 'end module m\n' + ) + + def test_warning_and_no_error(self): + import io + with tempfile.TemporaryDirectory() as d: + meta_path = os.path.join(d, 's.meta') + f90_path = os.path.join(d, 's.F90') + with open(meta_path, 'w') as fh: + fh.write(self._META) + with open(f90_path, 'w') as fh: + fh.write(self._F90) + stream = io.StringIO() + handler = logging.StreamHandler(stream) + handler.setLevel(logging.WARNING) + log = logging.getLogger('test_validator_optional_warn') + log.addHandler(handler) + log.setLevel(logging.WARNING) + try: + errs = validate([meta_path], [f90_path], logger=log) + finally: + log.removeHandler(handler) + self.assertEqual(errs, []) + self.assertIn("Optional Fortran argument 'b'", stream.getvalue()) + + +class _HostValidationFixture(unittest.TestCase): + """Shared scaffolding for host/ddt validation tests. + + Builds a tmpdir with one host metadata file and one Fortran source. + Subclasses set ``META`` and ``F90`` class attributes. + """ + + META: str = '' + F90: str = '' + META_NAME: str = 'host.meta' + F90_NAME: str = 'host.F90' + + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + self.meta_path = os.path.join(self.tmpdir, self.META_NAME) + self.f90_path = os.path.join(self.tmpdir, self.F90_NAME) + with open(self.meta_path, 'w') as fh: + fh.write(self.META) + with open(self.f90_path, 'w') as fh: + fh.write(self.F90) + + def tearDown(self): + import shutil + shutil.rmtree(self.tmpdir, ignore_errors=True) + + +class TestParseModulesAndDDTs(unittest.TestCase): + """Standalone parser-level tests beyond the doctest example.""" + + def test_module_var_after_use_only(self): + src = textwrap.dedent("""\ + module m + use kinds, only: kind_phys + implicit none + integer :: nlev + real(kind=kind_phys) :: cp + end module m + """) + mods = _parse_modules(src) + self.assertEqual(sorted(mods['m'].vars.keys()), ['cp', 'nlev']) + self.assertEqual(mods['m'].vars['cp'].type_, 'real') + self.assertEqual(mods['m'].vars['cp'].kind_, 'kind_phys') + + def test_subroutine_locals_excluded(self): + src = textwrap.dedent("""\ + module m + integer :: mod_var + contains + subroutine helper(x) + integer, intent(in) :: x + real :: local_only + end subroutine helper + end module m + """) + mods = _parse_modules(src) + self.assertIn('mod_var', mods['m'].vars) + self.assertNotIn('local_only', mods['m'].vars) + self.assertNotIn('x', mods['m'].vars) + + def test_ddt_block_components(self): + src = textwrap.dedent("""\ + module types + type :: physics_t + real(kind=kind_phys) :: tk(:,:) + integer :: nlay + logical :: has_water + end type physics_t + end module types + """) + mods = _parse_modules(src) + comps = mods['types'].ddts['physics_t'] + self.assertEqual(sorted(comps.keys()), ['has_water', 'nlay', 'tk']) + self.assertEqual(comps['tk'].rank, 2) + self.assertEqual(comps['tk'].kind_, 'kind_phys') + self.assertEqual(comps['has_water'].type_, 'logical') + + def test_ddt_with_attrs_on_type_decl(self): + src = textwrap.dedent("""\ + module types + type, public :: opaque_t + integer :: a + end type opaque_t + end module types + """) + mods = _parse_modules(src) + self.assertIn('opaque_t', mods['types'].ddts) + + def test_ddt_index_collects_across_modules(self): + src1 = ('module a\n type :: t1\n integer :: x\n end type t1\n' + 'end module a\n') + src2 = ('module b\n type :: t2\n real :: y\n end type t2\n' + 'end module b\n') + tmp = tempfile.mkdtemp() + try: + p1 = os.path.join(tmp, 'a.F90') + p2 = os.path.join(tmp, 'b.F90') + with open(p1, 'w') as fh: fh.write(src1) + with open(p2, 'w') as fh: fh.write(src2) + modules, ddt_index = _load_modules_tree([p1, p2]) + finally: + import shutil + shutil.rmtree(tmp, ignore_errors=True) + self.assertEqual(sorted(modules.keys()), ['a', 'b']) + self.assertEqual(sorted(ddt_index.keys()), ['t1', 't2']) + + +class TestBaseLocalName(unittest.TestCase): + + def test_plain(self): + self.assertEqual(_base_local_name('foo'), 'foo') + + def test_case_folded(self): + self.assertEqual(_base_local_name('Foo_Bar'), 'foo_bar') + + def test_strips_subscript(self): + self.assertEqual(_base_local_name('tk(:,:)'), 'tk') + + def test_strips_named_subscript(self): + self.assertEqual( + _base_local_name('dqdt(:,:,index_of_cloud)'), 'dqdt', + ) + + +class TestValidateHostCorrect(_HostValidationFixture): + """type=host with matching Fortran module → no errors.""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_host + type = host + [ccpp-arg-table] + name = my_host + type = host + [ nlev ] + standard_name = vertical_layer_dimension + long_name = number of layers + units = count + dimensions = () + type = integer + [ cp ] + standard_name = specific_heat_of_dry_air_at_constant_pressure + long_name = specific heat + units = J kg-1 K-1 + dimensions = () + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module my_host + implicit none + integer :: nlev + real(kind=kind_phys) :: cp + end module my_host + """) + + def test_no_errors(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(errs, []) + + +class TestValidateHostTypeMismatch(_HostValidationFixture): + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_host + type = host + [ccpp-arg-table] + name = my_host + type = host + [ cp ] + standard_name = specific_heat_of_dry_air_at_constant_pressure + long_name = specific heat + units = J kg-1 K-1 + dimensions = () + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module my_host + implicit none + integer :: cp + end module my_host + """) + + def test_type_error(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + # Type mismatch surfaces; an incidental kind mismatch may also + # surface (integer Fortran has no kind metadata). + type_errs = [e for e in errs if 'type mismatch' in e] + self.assertEqual(len(type_errs), 1, errs) + self.assertIn("cp", type_errs[0]) + + +class TestValidateHostKindMismatch(_HostValidationFixture): + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_host + type = host + [ccpp-arg-table] + name = my_host + type = host + [ cp ] + standard_name = specific_heat_of_dry_air_at_constant_pressure + long_name = specific heat + units = J kg-1 K-1 + dimensions = () + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module my_host + implicit none + real :: cp + end module my_host + """) + + def test_kind_error(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(len(errs), 1, errs) + self.assertIn("kind mismatch", errs[0]) + + +class TestValidateHostCharacterAssumedLength(_HostValidationFixture): + """A host character variable declared ``kind = len=*`` is rejected even + when the Fortran side has a concrete length (the metadata defines the + storage there, so assumed length is illegal).""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_host + type = host + [ccpp-arg-table] + name = my_host + type = host + [ scheme_name ] + standard_name = scheme_name + long_name = scheme name + units = none + dimensions = () + type = character | kind = len=* + """) + + F90 = textwrap.dedent("""\ + module my_host + implicit none + character(len=512) :: scheme_name + end module my_host + """) + + def test_assumed_length_error(self): + # Two complementary errors now fire: the definition-site rejection of + # len=* in host/DDT metadata, plus the exact-match inconsistency vs the + # concrete Fortran (character(len=512)). Both name scheme_name. + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertTrue( + any("concrete length" in e and "len=*" in e for e in errs), errs + ) + self.assertTrue(all("scheme_name" in e for e in errs), errs) + + +class TestValidateHostCharacterConcreteLengthOK(_HostValidationFixture): + """A concrete host character length matching the Fortran passes.""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_host + type = host + [ccpp-arg-table] + name = my_host + type = host + [ scheme_name ] + standard_name = scheme_name + long_name = scheme name + units = none + dimensions = () + type = character | kind = len=512 + """) + + F90 = textwrap.dedent("""\ + module my_host + implicit none + character(len=512) :: scheme_name + end module my_host + """) + + def test_concrete_length_ok(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(errs, []) + + +class TestValidateHostRankMismatch(_HostValidationFixture): + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_host + type = host + [ccpp-arg-table] + name = my_host + type = host + [ tk ] + standard_name = air_temperature + long_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module my_host + implicit none + real(kind=kind_phys) :: tk(:) + end module my_host + """) + + def test_rank_error(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(len(errs), 1, errs) + self.assertIn("rank mismatch", errs[0]) + + +class TestValidateHostMissingVar(_HostValidationFixture): + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_host + type = host + [ccpp-arg-table] + name = my_host + type = host + [ cp ] + standard_name = specific_heat_of_dry_air_at_constant_pressure + long_name = specific heat + units = J kg-1 K-1 + dimensions = () + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module my_host + implicit none + integer :: nlev + end module my_host + """) + + def test_missing_decl_error(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(len(errs), 1, errs) + self.assertIn("not found", errs[0]) + self.assertIn("cp", errs[0]) + + +class TestValidateHostMissingModule(_HostValidationFixture): + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_host + type = host + [ccpp-arg-table] + name = my_host + type = host + [ cp ] + standard_name = specific_heat_of_dry_air_at_constant_pressure + long_name = specific heat + units = J kg-1 K-1 + dimensions = () + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module different_module + implicit none + real(kind=kind_phys) :: cp + end module different_module + """) + + def test_module_not_found_error(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(len(errs), 1, errs) + self.assertIn("not found in any source file", errs[0]) + self.assertIn("my_host", errs[0]) + + +class TestValidateHostModuleNameOverride(_HostValidationFixture): + """module_name override in [ccpp-table-properties] honoured.""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_table + type = host + module_name = host_state + [ccpp-arg-table] + name = my_table + type = host + [ cp ] + standard_name = specific_heat_of_dry_air_at_constant_pressure + long_name = specific heat + units = J kg-1 K-1 + dimensions = () + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module host_state + implicit none + real(kind=kind_phys) :: cp + end module host_state + """) + + def test_no_errors_with_override(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(errs, []) + + +class TestValidateDDTCorrect(_HostValidationFixture): + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = physics_t + type = ddt + [ccpp-arg-table] + name = physics_t + type = ddt + [ tk ] + standard_name = air_temperature + long_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + [ nlay ] + standard_name = vertical_layer_dimension + long_name = number of layers + units = count + dimensions = () + type = integer + """) + + F90 = textwrap.dedent("""\ + module phys_types + implicit none + type :: physics_t + real(kind=kind_phys) :: tk(:,:) + integer :: nlay + end type physics_t + end module phys_types + """) + + META_NAME = 'physics_t.meta' + F90_NAME = 'phys_types.F90' + + def test_no_errors(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(errs, []) + + +class TestValidateDDTSlicedLocalName(_HostValidationFixture): + """A sliced metadata local_name expresses a reduced-rank view of a + higher-rank Fortran component. The rank check must consult the + subscript width — not just len(metadata.dimensions) — when computing + the expected Fortran rank. Mirrors the + end-to-end-tests/ddthost/test_host_data shape: rank-3 Fortran ``q`` + bound under TWO standard names — the bare ``q`` (3-D mixing ratio) + and the sliced ``q(:,:,index_of_water_vapor_specific_humidity)`` + (2-D view of one tracer).""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = physics_state + type = ddt + [ccpp-arg-table] + name = physics_state + type = ddt + [ q ] + standard_name = constituent_mixing_ratio + long_name = q + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_tracers) + type = real | kind = kind_phys + [ q(:,:,index_of_water_vapor_specific_humidity) ] + standard_name = water_vapor_specific_humidity + long_name = q water vapor + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module phys_types + implicit none + type :: physics_state + real(kind=kind_phys), dimension(:, :, :), allocatable :: q + end type physics_state + end module phys_types + """) + + META_NAME = 'physics_state.meta' + F90_NAME = 'phys_types.F90' + + def test_no_errors_for_sliced_and_bare_view(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(errs, []) + + +class TestValidateHostArrayInitialiserRank(_HostValidationFixture): + """Module-level parameter declarations with an array-constructor + initialiser (``(/ 'a', 'b', 'c' /)``) must not have the + initialiser's commas counted as dimension entries — that + regression made the rank-1 ``std_name_array`` on the + end-to-end-tests/advection fixture look like rank 3. Mirrors + that shape exactly.""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = test_host_data + type = host + [ccpp-arg-table] + name = test_host_data + type = host + [ num_consts ] + standard_name = number_of_constituents + units = count + dimensions = () + type = integer + [ std_name_array ] + standard_name = array_of_constituent_standard_names + units = none + dimensions = (number_of_constituents) + type = character | kind = len=32 + """) + + F90 = textwrap.dedent("""\ + module test_host_data + implicit none + integer, public, parameter :: num_consts = 3 + character(len=32), public, parameter :: std_name_array(num_consts) = (/ & + 'specific_humidity ', & + 'cloud_liquid_dry_mixing_ratio', & + 'cloud_ice_dry_mixing_ratio ' /) + end module test_host_data + """) + + META_NAME = 'test_host_data.meta' + F90_NAME = 'test_host_data.F90' + + def test_no_errors(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(errs, []) + + +class TestValidateDDTSlicedRankStillCaught(_HostValidationFixture): + """A truly wrong rank under a sliced spelling is still caught. The + Fortran ``q`` here is rank 2 instead of the rank-3 the metadata + implies — the subscript declares 3 entries (``(:,:,index_of_X)``) + but Fortran only carries 2.""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = physics_state + type = ddt + [ccpp-arg-table] + name = physics_state + type = ddt + [ q(:,:,index_of_water_vapor_specific_humidity) ] + standard_name = water_vapor_specific_humidity + long_name = q water vapor + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module phys_types + implicit none + type :: physics_state + real(kind=kind_phys), dimension(:, :), allocatable :: q + end type physics_state + end module phys_types + """) + + META_NAME = 'physics_state.meta' + F90_NAME = 'phys_types.F90' + + def test_rank_mismatch_surfaces(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + rank_errs = [e for e in errs if 'rank mismatch' in e] + self.assertEqual(len(rank_errs), 1, errs) + self.assertIn("q(:,:,index_of_water_vapor_specific_humidity)", + rank_errs[0]) + self.assertIn("Fortran declares rank 2", rank_errs[0]) + + +class TestValidateDDTComponentMismatch(_HostValidationFixture): + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = physics_t + type = ddt + [ccpp-arg-table] + name = physics_t + type = ddt + [ tk ] + standard_name = air_temperature + long_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module phys_types + implicit none + type :: physics_t + integer :: tk(:,:) + end type physics_t + end module phys_types + """) + + META_NAME = 'physics_t.meta' + F90_NAME = 'phys_types.F90' + + def test_type_error(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + type_errs = [e for e in errs if 'type mismatch' in e] + self.assertEqual(len(type_errs), 1, errs) + self.assertIn("tk", type_errs[0]) + + +class TestValidateDDTMissingComponent(_HostValidationFixture): + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = physics_t + type = ddt + [ccpp-arg-table] + name = physics_t + type = ddt + [ tk ] + standard_name = air_temperature + long_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module phys_types + implicit none + type :: physics_t + integer :: nlay + end type physics_t + end module phys_types + """) + + META_NAME = 'physics_t.meta' + F90_NAME = 'phys_types.F90' + + def test_missing_component_error(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(len(errs), 1, errs) + self.assertIn("not found as a component", errs[0]) + self.assertIn("tk", errs[0]) + + +class TestValidateDDTMissingType(_HostValidationFixture): + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = physics_t + type = ddt + [ccpp-arg-table] + name = physics_t + type = ddt + [ tk ] + standard_name = air_temperature + long_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module phys_types + implicit none + type :: different_type + real(kind=kind_phys) :: tk(:,:) + end type different_type + end module phys_types + """) + + META_NAME = 'physics_t.meta' + F90_NAME = 'phys_types.F90' + + def test_type_not_found_error(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(len(errs), 1, errs) + self.assertIn("not found as a derived-type definition", errs[0]) + self.assertIn("physics_t", errs[0]) + + +class TestValidateControlSilentSkip(_HostValidationFixture): + """type=control tables are silent-skipped (no Fortran backs control vars).""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_control + type = control + [ccpp-arg-table] + name = my_control + type = control + [ suite_name ] + standard_name = suite_name + long_name = suite name + units = none + dimensions = () + type = character | kind = len=* + [ errcode ] + standard_name = ccpp_error_code + long_name = error code + units = 1 + dimensions = () + type = integer + """) + + # F90 is irrelevant — control vars never resolved against source. + F90 = textwrap.dedent("""\ + module placeholder + implicit none + end module placeholder + """) + + def test_no_errors_for_control(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(errs, []) + + +class TestValidateSchemeInHostFilesRejected(_HostValidationFixture): + """type=scheme passed via --host-files is a hard error.""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ a ] + standard_name = horizontal_dimension + long_name = horizontal dim + units = count + dimensions = () + type = integer + intent = in + """) + + F90 = textwrap.dedent("""\ + module placeholder + implicit none + end module placeholder + """) + + def test_raises_ccpperror(self): + with self.assertRaises(CCPPError) as cm: + validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertIn("--host-files", str(cm.exception)) + self.assertIn("my_scheme", str(cm.exception)) + + +class TestValidateNonSchemeInSchemeFilesRejected(_HostValidationFixture): + """type=host / type=control / type=ddt in --scheme-files is rejected. + + Symmetric to the scheme-in-host-files rejection above: each CLI + flag has a single responsibility so misclassified .meta files fail + fast with a clear pointer at the right flag. + """ + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_host + type = host + [ccpp-arg-table] + name = my_host + type = host + [ nlev ] + standard_name = vertical_layer_dimension + long_name = number of layers + units = count + dimensions = () + type = integer + """) + + F90 = textwrap.dedent("""\ + module my_host + implicit none + integer :: nlev + end module my_host + """) + + def test_host_in_scheme_files_rejected(self): + with self.assertRaises(CCPPError) as cm: + validate([self.meta_path], [self.f90_path]) + msg = str(cm.exception) + self.assertIn("--scheme-files", msg) + self.assertIn("--host-files", msg) + self.assertIn("my_host", msg) + self.assertIn("type = host", msg) + + def test_control_in_scheme_files_rejected(self): + # Replace the host fixture's meta with a control-typed one and + # check the same rejection path covers control too. + with open(self.meta_path, 'w') as fh: + fh.write(textwrap.dedent("""\ + [ccpp-table-properties] + name = my_control + type = control + [ccpp-arg-table] + name = my_control + type = control + [ suite_name ] + standard_name = suite_name + long_name = suite name + units = none + dimensions = () + type = character | kind = len=* + """)) + with self.assertRaises(CCPPError) as cm: + validate([self.meta_path], [self.f90_path]) + msg = str(cm.exception) + self.assertIn("my_control", msg) + self.assertIn("type = control", msg) + + def test_suite_in_scheme_files_rejected(self): + # type = suite is the third rejected type (host / control / suite). + with open(self.meta_path, 'w') as fh: + fh.write(textwrap.dedent("""\ + [ccpp-table-properties] + name = my_suite + type = suite + [ccpp-arg-table] + name = my_suite + type = suite + """)) + with self.assertRaises(CCPPError) as cm: + validate([self.meta_path], [self.f90_path]) + msg = str(cm.exception) + self.assertIn("my_suite", msg) + self.assertIn("type = suite", msg) + + +class TestValidateSchemeWithCoLocatedDDT(_HostValidationFixture): + """A scheme metadata file may contain its own type = ddt tables — + schemes routinely co-locate the DDTs they define (e.g. radiation + schemes carrying ty_rad_lw / ty_rad_sw type definitions in the same + .meta as the scheme phase blocks). Both the scheme phase signature + AND the DDT components must be validated.""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ workspace ] + standard_name = scheme_workspace + long_name = workspace + units = none + dimensions = () + type = ty_ws + intent = inout + [ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character | kind = len=* + intent = out + [ errflg ] + standard_name = ccpp_error_code + long_name = error code + units = 1 + dimensions = () + type = integer + intent = out + [ccpp-table-properties] + name = ty_ws + type = ddt + [ccpp-arg-table] + name = ty_ws + type = ddt + [ nlay ] + standard_name = vertical_layer_dimension + long_name = number of layers + units = count + dimensions = () + type = integer + [ tk ] + standard_name = air_temperature + long_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module my_scheme + implicit none + type :: ty_ws + integer :: nlay + real(kind=kind_phys) :: tk(:,:) + end type ty_ws + contains + subroutine my_scheme_run(workspace, errmsg, errflg) + type(ty_ws), intent(inout) :: workspace + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + end subroutine my_scheme_run + end module my_scheme + """) + + META_NAME = 'my_scheme.meta' + F90_NAME = 'my_scheme.F90' + + def test_no_errors_when_both_match(self): + errs = validate([self.meta_path], [self.f90_path]) + self.assertEqual(errs, []) + + def test_ddt_component_mismatch_surfaces(self): + # Break the DDT component type and confirm the validator + # catches it (proves the DDT pass actually runs on scheme files). + broken_f90 = textwrap.dedent("""\ + module my_scheme + implicit none + type :: ty_ws + integer :: nlay + integer :: tk(:,:) + end type ty_ws + contains + subroutine my_scheme_run(workspace, errmsg, errflg) + type(ty_ws), intent(inout) :: workspace + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + end subroutine my_scheme_run + end module my_scheme + """) + with open(self.f90_path, 'w') as fh: + fh.write(broken_f90) + errs = validate([self.meta_path], [self.f90_path]) + type_errs = [e for e in errs if 'type mismatch' in e] + self.assertEqual(len(type_errs), 1, errs) + self.assertIn("tk", type_errs[0]) + self.assertIn("ty_ws", type_errs[0]) + + +class TestValidateRequiresAtLeastOneInputs(unittest.TestCase): + """validate() must error when neither scheme_files nor host_files is supplied.""" + + def test_both_empty_raises(self): + with self.assertRaises(CCPPError) as cm: + validate([], [], host_files=[]) + self.assertIn("at least one", str(cm.exception)) + self.assertIn("--scheme-files", str(cm.exception)) + self.assertIn("--host-files", str(cm.exception)) + + def test_default_host_files_kwarg_also_raises(self): + # Same condition reached via the default value of host_files. + with self.assertRaises(CCPPError): + validate([], []) + + +def load_tests(loader, tests, ignore): + tests.addTests(doctest.DocTestSuite(val_mod)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_variable_resolver.py b/unit-tests/test_variable_resolver.py new file mode 100644 index 00000000..072d79d8 --- /dev/null +++ b/unit-tests/test_variable_resolver.py @@ -0,0 +1,1335 @@ +"""Unit tests for metadata.variable_resolver. + +Covers: +- Helper functions (_ddt_typename, _is_intrinsic, _is_external, _is_known_ddt, + _instance_subscript, _build_ddt_index) +- HostVarEntry construction and properties +- _flatten_ddt_instance (single-level and nested) +- build_flat_host_dict (plain vars, DDT expansion, control vars, duplicates) +- SchemeStore (build_from, has_scheme, phases_for, variables_for, duplicates) +""" + +import os +import sys +import unittest +import doctest + +# Path setup is handled by conftest.py (pytest) or run_tests.py; the imports +# below are enough for direct invocation via this file's __main__ block. +from metadata.metadata_table import _parse_lines, MetadataTable, MetaVar +from metadata.parse_tools import ParseContext, CCPPError +from metadata.variable_resolver import ( + _ddt_typename, + _is_intrinsic, + _is_external, + _is_known_ddt, + _instance_subscript, + _build_ddt_index, + _flatten_ddt_instance, + build_ddt_module_map, + HostVarEntry, + build_flat_host_dict, + SchemeStore, +) + +# --------------------------------------------------------------------------- +# Sample file directory +# --------------------------------------------------------------------------- +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_SAMPLES_DIR = os.path.join(_TESTS_DIR, 'sample_files') + + +def _sample(name: str) -> str: + return os.path.join(_SAMPLES_DIR, name) + + +def _parse_file(name: str): + """Parse a sample metadata file and return its tables.""" + from metadata.metadata_table import parse_metadata_file + return parse_metadata_file(_sample(name)) + + +def _ctx(lineno: int = 0, filename: str = 'test.meta') -> ParseContext: + return ParseContext(linenum=lineno, filename=filename) + + +def _make_simple_var(local_name: str, std_name: str, units: str = '1', + dims: str = '()', type_: str = 'integer', + kind: str = '') -> MetaVar: + """Quick helper to build a validated MetaVar.""" + ctx = _ctx() + v = MetaVar(local_name, ctx) + v.set_attr('standard_name', std_name, ctx) + v.set_attr('units', units, ctx) + v.set_attr('dimensions', dims, ctx) + v.set_attr('type', type_, ctx) + if kind: + v.set_attr('kind', kind, ctx) + return v + + +def _make_ddt_instance_var(local_name: str, std_name: str, type_name: str, + dims: str = '()') -> MetaVar: + """Build a DDT instance MetaVar with the given dimensions.""" + ctx = _ctx() + v = MetaVar(local_name, ctx) + v.set_attr('standard_name', std_name, ctx) + v.set_attr('units', 'none', ctx) + v.set_attr('dimensions', dims, ctx) + v.set_attr('type', type_name, ctx) + return v + + +######################################################################## +# Tests: helper functions +######################################################################## + +class TestDdtTypename(unittest.TestCase): + + def test_plain_name(self): + self.assertEqual(_ddt_typename('gfs_statein_type'), 'gfs_statein_type') + + def test_type_paren(self): + self.assertEqual(_ddt_typename('type(gfs_statein_type)'), 'gfs_statein_type') + + def test_type_paren_uppercase(self): + self.assertEqual(_ddt_typename('TYPE(MY_DDT)'), 'MY_DDT') + + def test_type_paren_whitespace(self): + self.assertEqual(_ddt_typename('type( my_type )'), 'my_type') + + def test_intrinsic_passthrough(self): + self.assertEqual(_ddt_typename('real'), 'real') + + def test_external_passthrough(self): + self.assertEqual(_ddt_typename('external:mpi_f08:mpi_comm'), + 'external:mpi_f08:mpi_comm') + + +class TestIsIntrinsic(unittest.TestCase): + + def test_real(self): + self.assertTrue(_is_intrinsic('real')) + + def test_integer(self): + self.assertTrue(_is_intrinsic('integer')) + + def test_character(self): + self.assertTrue(_is_intrinsic('character')) + + def test_logical(self): + self.assertTrue(_is_intrinsic('logical')) + + def test_complex(self): + self.assertTrue(_is_intrinsic('complex')) + + def test_ddt_name(self): + self.assertFalse(_is_intrinsic('gfs_statein_type')) + + def test_external_syntax(self): + self.assertFalse(_is_intrinsic('external:mpi_f08:mpi_comm')) + + +class TestIsExternal(unittest.TestCase): + + def test_external_lowercase(self): + self.assertTrue(_is_external('external:mpi_f08:mpi_comm')) + + def test_external_uppercase(self): + self.assertTrue(_is_external('EXTERNAL:mpi_f08:mpi_comm')) + + def test_real(self): + self.assertFalse(_is_external('real')) + + def test_ddt_name(self): + self.assertFalse(_is_external('my_ddt_type')) + + +class TestIsKnownDdt(unittest.TestCase): + + def _idx(self, name: str) -> dict: + ctx = _ctx() + tbl = MetadataTable(name, 'ddt', 'f.meta', ctx) + return {name: tbl} + + def test_known_ddt(self): + self.assertTrue(_is_known_ddt('my_type', self._idx('my_type'))) + + def test_type_paren_form(self): + self.assertTrue(_is_known_ddt('type(my_type)', self._idx('my_type'))) + + def test_unknown_name(self): + self.assertFalse(_is_known_ddt('other_type', self._idx('my_type'))) + + def test_intrinsic_not_ddt(self): + self.assertFalse(_is_known_ddt('real', self._idx('my_type'))) + + def test_external_not_ddt(self): + self.assertFalse(_is_known_ddt('external:mpi_f08:mpi_comm', self._idx('my_type'))) + + def test_empty_index(self): + self.assertFalse(_is_known_ddt('my_type', {})) + + +class TestInstanceSubscript(unittest.TestCase): + + def test_number_of_instances(self): + v = _make_ddt_instance_var('gs', 'gst', 'gs_type', '(number_of_instances)') + self.assertEqual(_instance_subscript(v), '(instance_number)') + + def test_number_of_threads(self): + """Registered scalar-index dim 'number_of_threads' pairs with + the host's 'thread_number' control variable. Regression for + the per-thread DDT-container pattern (e.g. SCM's + physics%Interstitial(thread_number)).""" + v = _make_ddt_instance_var('inst', 'inst_std', 'gs_type', + '(number_of_threads)') + self.assertEqual(_instance_subscript(v), '(thread_number)') + + def test_multiple_registered_dims(self): + """A DDT instance with two registered scalar-index dims emits + both index placeholders in declared order.""" + v = _make_ddt_instance_var('foo', 'foo_std', 'foo_type', + '(number_of_instances, number_of_threads)') + self.assertEqual( + _instance_subscript(v), + '(instance_number, thread_number)', + ) + + def test_scalar_no_subscript(self): + v = _make_ddt_instance_var('gs', 'gst', 'gs_type', '()') + self.assertEqual(_instance_subscript(v), '') + + def test_horizontal_dim_no_subscript(self): + v = _make_simple_var('x', 'air_temperature', 'K', + '(horizontal_dimension, vertical_layer_dimension)', + 'real', 'kind_phys') + self.assertEqual(_instance_subscript(v), '') + + +######################################################################## +# Tests: HostVarEntry +######################################################################## + +class TestHostVarEntry(unittest.TestCase): + + def _make(self, stdname='air_temperature', local='temp', + path='temp', module='mymod'): + return HostVarEntry( + stdname, local, path, module, + 'real', 'kind_phys', 'K', + ['horizontal_dimension', 'vertical_layer_dimension'], + False, False, '' + ) + + def test_basic_attributes(self): + e = self._make() + self.assertEqual(e.standard_name, 'air_temperature') + self.assertEqual(e.local_name, 'temp') + self.assertEqual(e.access_path, 'temp') + self.assertEqual(e.module_name, 'mymod') + self.assertEqual(e.type, 'real') + self.assertEqual(e.kind, 'kind_phys') + self.assertEqual(e.units, 'K') + self.assertEqual(e.dimensions, + ['horizontal_dimension', 'vertical_layer_dimension']) + self.assertFalse(e.protected) + self.assertFalse(e.optional) + self.assertEqual(e.active, '') + + def test_is_control_false(self): + e = self._make() + self.assertFalse(e.is_control) + + def test_is_control_true(self): + e = HostVarEntry('thread_number', 'thread_num', 'thread_num', None, + 'integer', '', '1', [], False, False, '') + self.assertTrue(e.is_control) + + def test_repr(self): + e = self._make() + self.assertIn('air_temperature', repr(e)) + self.assertIn('temp', repr(e)) + + def test_dimensions_are_copied(self): + dims = ['horizontal_dimension'] + e = HostVarEntry('x', 'x', 'x', 'mod', 'integer', '', '1', + dims, False, False, '') + dims.append('extra') + self.assertEqual(len(e.dimensions), 1) + + def test_equality_by_standard_name(self): + e1 = self._make('air_temperature') + e2 = self._make('air_temperature') + e3 = self._make('pressure') + self.assertEqual(e1, e2) + self.assertNotEqual(e1, e3) + + def test_hash(self): + e1 = self._make('air_temperature') + e2 = self._make('air_temperature') + self.assertEqual(hash(e1), hash(e2)) + + +######################################################################## +# Tests: _build_ddt_index +######################################################################## + +class TestBuildDdtIndex(unittest.TestCase): + + def test_single_table(self): + ctx = _ctx() + tbl = MetadataTable('gfs_statein_type', 'ddt', 'f.meta', ctx) + idx = _build_ddt_index([tbl]) + self.assertIn('gfs_statein_type', idx) + self.assertIs(idx['gfs_statein_type'], tbl) + + def test_multiple_tables(self): + ctx = _ctx() + t1 = MetadataTable('type_a', 'ddt', 'a.meta', ctx) + t2 = MetadataTable('type_b', 'ddt', 'b.meta', ctx) + idx = _build_ddt_index([t1, t2]) + self.assertEqual(set(idx.keys()), {'type_a', 'type_b'}) + + def test_empty(self): + self.assertEqual(_build_ddt_index([]), {}) + + def test_from_file(self): + tables = _parse_file('ddt_simple.meta') + idx = _build_ddt_index(tables) + self.assertIn('gfs_statein_type', idx) + + +######################################################################## +# Tests: build_ddt_module_map +######################################################################## + +class TestBuildDdtModuleMap(unittest.TestCase): + + def test_ddt_co_located_with_scheme(self): + ctx = _ctx() + ddt_tbl = MetadataTable('vmr_type', 'ddt', 'make_ddt.meta', ctx) + sch_tbl = MetadataTable('make_ddt', 'scheme', 'make_ddt.meta', ctx) + result = build_ddt_module_map([ddt_tbl, sch_tbl]) + self.assertEqual(result, {'vmr_type': 'make_ddt'}) + + def test_ddt_co_located_with_host(self): + ctx = _ctx() + ddt_tbl = MetadataTable('payload_t', 'ddt', 'host.meta', ctx) + host_tbl = MetadataTable('my_host', 'host', 'host.meta', ctx) + result = build_ddt_module_map([ddt_tbl, host_tbl]) + self.assertEqual(result, {'payload_t': 'my_host'}) + + def test_ddt_alone_in_file_skipped(self): + ctx = _ctx() + orphan = MetadataTable('lonely_t', 'ddt', 'lonely.meta', ctx) + self.assertEqual(build_ddt_module_map([orphan]), {}) + + def test_multiple_files(self): + ctx = _ctx() + d1 = MetadataTable('t1', 'ddt', 'a.meta', ctx) + s1 = MetadataTable('mod_a', 'scheme', 'a.meta', ctx) + d2 = MetadataTable('t2', 'ddt', 'b.meta', ctx) + h2 = MetadataTable('mod_b', 'host', 'b.meta', ctx) + result = build_ddt_module_map([d1, s1, d2, h2]) + self.assertEqual(result, {'t1': 'mod_a', 't2': 'mod_b'}) + + def test_multiple_ddts_in_one_file(self): + ctx = _ctx() + d1 = MetadataTable('inner', 'ddt', 'm.meta', ctx) + d2 = MetadataTable('outer', 'ddt', 'm.meta', ctx) + s = MetadataTable('host_mod', 'host', 'm.meta', ctx) + result = build_ddt_module_map([d1, d2, s]) + self.assertEqual(result, {'inner': 'host_mod', 'outer': 'host_mod'}) + + def test_empty(self): + self.assertEqual(build_ddt_module_map([]), {}) + + # ---- Precedence cases for the DDT module map ------------------------ + # Truth table (see build_ddt_module_map docstring): + # + # DDT.module_name | co-located resolved | Result + # ------------------|---------------------|-------- + # X (set) | Y (any) | X (DDT wins) + # unset | Y (any) | Y + # X (set) | (no co-located tbl) | X + # unset | (no co-located tbl) | (skipped) + # + # The "co-located resolved" column is itself the same + # ``module_name or table_name`` rule used by build_flat_host_dict. + + def test_explicit_module_name_used_when_no_colocated_table(self): + """Real-world fixture: CCPP-physics ``radsw_param.meta`` declares + ``cmpfsw_type`` in a file containing only DDT tables, where the + Fortran module name differs from any table name and is supplied + via ``module_name = …`` in ``[ccpp-table-properties]``. Without + this resolution path the suite_types emitter can't find the + module and raises CCPPError on pointer-wrapper generation.""" + ctx = _ctx() + tbl = MetadataTable('cmpfsw_type', 'ddt', 'radsw_param.meta', ctx) + tbl.module_name = 'module_radsw_parameters' + self.assertEqual( + build_ddt_module_map([tbl]), + {'cmpfsw_type': 'module_radsw_parameters'}, + ) + + def test_ddt_module_name_overrides_colocated_table_name(self): + """``module_name`` on a DDT table beats the implicit co-located + scheme/host name — the DDT may genuinely live in a different + Fortran module than the scheme its .meta is paired with.""" + ctx = _ctx() + ddt = MetadataTable('cmpfsw_type', 'ddt', 'rad.meta', ctx) + ddt.module_name = 'module_radsw_parameters' + sch = MetadataTable('radsw_main', 'scheme', 'rad.meta', ctx) + result = build_ddt_module_map([ddt, sch]) + self.assertEqual(result['cmpfsw_type'], 'module_radsw_parameters') + + def test_ddt_module_name_wins_over_colocated_module_name(self): + """Both the DDT and a co-located scheme declare module_name; the + DDT's takes precedence (most-specific-wins). Documents the + truth-table row "X / Y / X".""" + ctx = _ctx() + ddt = MetadataTable('cmpfsw_type', 'ddt', 'rad.meta', ctx) + ddt.module_name = 'module_radsw_parameters' + sch = MetadataTable('radsw_main', 'scheme', 'rad.meta', ctx) + sch.module_name = 'mod_radsw_main' + result = build_ddt_module_map([ddt, sch]) + self.assertEqual(result['cmpfsw_type'], 'module_radsw_parameters') + + def test_colocated_module_name_used_when_ddt_has_none(self): + """When the DDT has no ``module_name`` but the co-located + scheme/host carries one, the co-located ``module_name`` + (NOT its table name) wins. Documents "unset / Y / Y" where + Y comes from the co-located override. This is the bug we + fixed when refactoring build_ddt_module_map — the old code + used the co-located table_name and silently ignored its + own module_name override.""" + ctx = _ctx() + ddt = MetadataTable('inner_t', 'ddt', 'a.meta', ctx) + # No DDT override. + sch = MetadataTable('scheme_a', 'scheme', 'a.meta', ctx) + sch.module_name = 'mod_a' # Fortran module differs from table name + result = build_ddt_module_map([ddt, sch]) + self.assertEqual(result['inner_t'], 'mod_a') + + def test_colocated_table_name_used_when_neither_has_override(self): + """When neither carries module_name, the implicit "module = table + name" convention applies to the co-located scheme/host.""" + ctx = _ctx() + ddt = MetadataTable('inner_t', 'ddt', 'a.meta', ctx) + sch = MetadataTable('scheme_a', 'scheme', 'a.meta', ctx) + result = build_ddt_module_map([ddt, sch]) + self.assertEqual(result['inner_t'], 'scheme_a') + + +######################################################################## +# Tests: _flatten_ddt_instance +######################################################################## + +class TestFlattenDdtInstance(unittest.TestCase): + + def _load_ddt_gfs(self): + return _build_ddt_index(_parse_file('ddt_simple.meta')) + + def test_scalar_instance_paths(self): + """Scalar DDT instance (no dimensions) → field paths with % separator.""" + idx = self._load_ddt_gfs() + var = _make_ddt_instance_var('gfs_statein', 'gfs_statein', 'gfs_statein_type') + entries = _flatten_ddt_instance(var, 'CCPP_data', idx) + std_names = {e.standard_name for e in entries} + self.assertIn('gfs_statein', std_names) # instance itself + self.assertIn('geopotential_at_interface', std_names) + self.assertIn('geopotential', std_names) + + def test_scalar_instance_access_paths(self): + idx = self._load_ddt_gfs() + var = _make_ddt_instance_var('gfs_statein', 'gfs_statein', 'gfs_statein_type') + entries = {e.standard_name: e + for e in _flatten_ddt_instance(var, 'CCPP_data', idx)} + self.assertEqual(entries['geopotential_at_interface'].access_path, + 'gfs_statein%phii') + self.assertEqual(entries['geopotential'].access_path, + 'gfs_statein%phil') + + def test_arrayed_instance_subscript(self): + """DDT instance with instance dimension → access path has (instance_number).""" + idx = self._load_ddt_gfs() + var = _make_ddt_instance_var('gfs_statein', 'gfs_statein', + 'gfs_statein_type', '(number_of_instances)') + entries = {e.standard_name: e + for e in _flatten_ddt_instance(var, 'CCPP_data', idx)} + self.assertEqual(entries['geopotential_at_interface'].access_path, + 'gfs_statein(instance_number)%phii') + + def test_module_name_propagated(self): + idx = self._load_ddt_gfs() + var = _make_ddt_instance_var('gfs_statein', 'gfs_statein', 'gfs_statein_type') + for entry in _flatten_ddt_instance(var, 'GFS_typedefs', idx): + self.assertEqual(entry.module_name, 'GFS_typedefs') + + def test_unknown_ddt_type_raises(self): + idx = {} # empty — type not found + var = _make_ddt_instance_var('gs', 'gs', 'unknown_type') + with self.assertRaises(CCPPError) as cm: + _flatten_ddt_instance(var, 'mod', idx) + self.assertIn('unknown_type', str(cm.exception)) + + def test_field_type_and_kind(self): + idx = self._load_ddt_gfs() + var = _make_ddt_instance_var('gfs_statein', 'gfs_statein', 'gfs_statein_type') + entries = {e.standard_name: e + for e in _flatten_ddt_instance(var, 'CCPP_data', idx)} + phii = entries['geopotential_at_interface'] + self.assertEqual(phii.type, 'real') + self.assertEqual(phii.kind, 'kind_phys') + self.assertEqual(phii.units, 'm2 s-2') + + def test_prefix_propagated(self): + """access_prefix is prepended to all paths (used in nested DDT recursion).""" + idx = self._load_ddt_gfs() + var = _make_ddt_instance_var('inner', 'inner', 'gfs_statein_type') + entries = {e.standard_name: e + for e in _flatten_ddt_instance( + var, 'mod', idx, access_prefix='outer%')} + self.assertEqual(entries['geopotential_at_interface'].access_path, + 'outer%inner%phii') + + def test_max_depth_guard(self): + """A deeply nested structure beyond max_depth raises CCPPError.""" + idx = self._load_ddt_gfs() + var = _make_ddt_instance_var('x', 'x', 'gfs_statein_type') + with self.assertRaises(CCPPError) as cm: + _flatten_ddt_instance(var, 'mod', idx, depth=10, max_depth=8) + self.assertIn('depth', str(cm.exception)) + + +class TestFlattenNestedDdt(unittest.TestCase): + """End-to-end nested DDT flattening using sample files.""" + + def _load_nested(self): + outer_tables = _parse_file('ddt_nested_outer.meta') + inner_tables = _parse_file('ddt_nested_inner.meta') + return _build_ddt_index(outer_tables + inner_tables) + + def test_nested_field_standard_names(self): + idx = self._load_nested() + host_tables = _parse_file('host_with_nested_ddt.meta') + var = host_tables[0].sections()[0].variables[0] + entries = {e.standard_name: e + for e in _flatten_ddt_instance(var, 'nested_host_mod', idx)} + expected = { + 'outer_ddt_instance', # the outer DDT instance itself + 'outer_scalar_field', # plain field on the outer DDT + 'inner_ddt_instance', # the inner DDT instance (as a field) + 'inner_real_value', # field of the inner DDT + 'inner_integer_flag', # field of the inner DDT + } + self.assertEqual(set(entries.keys()), expected) + + def test_nested_access_paths(self): + idx = self._load_nested() + host_tables = _parse_file('host_with_nested_ddt.meta') + var = host_tables[0].sections()[0].variables[0] + entries = {e.standard_name: e + for e in _flatten_ddt_instance(var, 'nested_host_mod', idx)} + self.assertEqual(entries['outer_scalar_field'].access_path, + 'outer_inst%scalar_field') + self.assertEqual(entries['inner_real_value'].access_path, + 'outer_inst%inner_ddt%inner_value') + self.assertEqual(entries['inner_integer_flag'].access_path, + 'outer_inst%inner_ddt%inner_flag') + + def test_nested_module_name(self): + idx = self._load_nested() + host_tables = _parse_file('host_with_nested_ddt.meta') + var = host_tables[0].sections()[0].variables[0] + for entry in _flatten_ddt_instance(var, 'nested_host_mod', idx): + self.assertEqual(entry.module_name, 'nested_host_mod') + + +######################################################################## +# Tests: build_flat_host_dict +######################################################################## + +class TestBuildFlatHostDict(unittest.TestCase): + + def test_plain_host_vars(self): + host_tables = _parse_file('host_simple.meta') + d = build_flat_host_dict(host_tables, [], []) + self.assertIn('horizontal_dimension', d) + self.assertIn('vertical_layer_dimension', d) + # number_of_instances is now a control-table var (paired with + # instance_number), so it is NOT in a host-only dictionary. + self.assertNotIn('number_of_instances', d) + # loop bounds and error vars live in the control table, not the host table + self.assertNotIn('horizontal_loop_begin', d) + self.assertNotIn('horizontal_loop_end', d) + self.assertEqual(len(d), 2) + + def test_plain_host_access_paths(self): + host_tables = _parse_file('host_simple.meta') + d = build_flat_host_dict(host_tables, [], []) + # For plain vars the access path equals the local name. + self.assertEqual(d['horizontal_dimension'].access_path, 'ncols') + self.assertEqual(d['vertical_layer_dimension'].access_path, 'nlev') + + def test_plain_host_module_names(self): + host_tables = _parse_file('host_simple.meta') + d = build_flat_host_dict(host_tables, [], []) + for entry in d.values(): + self.assertEqual(entry.module_name, 'physics_data') + self.assertFalse(entry.is_control) + + def test_host_module_name_override(self): + """A ``type=host`` table that declares ``module_name`` in its + ``[ccpp-table-properties]`` should override the default convention + (module name = table name) so subsequent USE statements target the + actual Fortran module.""" + src = ''' +[ccpp-table-properties] + name = host_data + type = host + module_name = mod_host_data + +[ccpp-arg-table] + name = host_data + type = host +[ ncols ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +''' + tables = _parse_lines(src.splitlines(keepends=True), 'h.meta') + d = build_flat_host_dict(tables, [], []) + self.assertEqual(d['horizontal_dimension'].module_name, 'mod_host_data') + + def test_host_module_name_defaults_to_table_name(self): + """Without ``module_name`` the module defaults to the table name.""" + src = ''' +[ccpp-table-properties] + name = host_data + type = host + +[ccpp-arg-table] + name = host_data + type = host +[ ncols ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +''' + tables = _parse_lines(src.splitlines(keepends=True), 'h.meta') + d = build_flat_host_dict(tables, [], []) + self.assertEqual(d['horizontal_dimension'].module_name, 'host_data') + + def test_control_vars_no_module(self): + ctrl_tables = _parse_file('control_simple.meta') + d = build_flat_host_dict([], ctrl_tables, []) + self.assertIn('suite_name', d) + self.assertIn('group_name', d) + self.assertIn('thread_number', d) + self.assertIn('number_of_physics_threads', d) + for entry in d.values(): + self.assertIsNone(entry.module_name) + self.assertTrue(entry.is_control) + + def test_ddt_instance_expansion(self): + host_tables = _parse_file('host_with_ddt_instance.meta') + ddt_tables = _parse_file('ddt_simple.meta') + d = build_flat_host_dict(host_tables, [], ddt_tables) + # DDT instance itself + two fields + self.assertIn('gfs_statein', d) + self.assertIn('geopotential_at_interface', d) + self.assertIn('geopotential', d) + + def test_ddt_instance_subscript_in_path(self): + """DDT instance with (number_of_instances) → (instance_number) in path.""" + host_tables = _parse_file('host_with_ddt_instance.meta') + ddt_tables = _parse_file('ddt_simple.meta') + d = build_flat_host_dict(host_tables, [], ddt_tables) + self.assertEqual(d['geopotential_at_interface'].access_path, + 'gfs_statein(instance_number)%phii') + self.assertEqual(d['geopotential'].access_path, + 'gfs_statein(instance_number)%phil') + + def test_ddt_instance_module_name(self): + host_tables = _parse_file('host_with_ddt_instance.meta') + ddt_tables = _parse_file('ddt_simple.meta') + d = build_flat_host_dict(host_tables, [], ddt_tables) + self.assertEqual(d['geopotential_at_interface'].module_name, 'CCPP_data') + + def test_host_and_control_combined(self): + host_tables = _parse_file('host_simple.meta') + ctrl_tables = _parse_file('control_simple.meta') + d = build_flat_host_dict(host_tables, ctrl_tables, []) + self.assertIn('horizontal_dimension', d) + self.assertIn('suite_name', d) + self.assertFalse(d['horizontal_dimension'].is_control) + self.assertTrue(d['suite_name'].is_control) + + def test_duplicate_standard_name_raises(self): + """Same standard name in two host tables must raise CCPPError. + The message must include both access paths so the user can see + which two declarations collide.""" + src = ''' +[ccpp-table-properties] + name = mod_a + type = host +[ccpp-arg-table] + name = mod_a + type = host +[ im ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +''' + tables_a = _parse_lines(src.splitlines(keepends=True), 'a.meta') + tables_b = _parse_lines(src.replace('mod_a', 'mod_b') + .splitlines(keepends=True), 'b.meta') + with self.assertRaises(CCPPError) as cm: + build_flat_host_dict(tables_a + tables_b, [], []) + msg = str(cm.exception) + self.assertIn('horizontal_dimension', msg) + # Both module names must appear so the user can locate the duplicates. + self.assertIn('mod_a', msg) + self.assertIn('mod_b', msg) + # And both access paths. + self.assertIn('access path', msg) + + def test_duplicate_ddt_component_names_path_collision(self): + """Two sibling DDT instances of the same type inside one parent + DDT cause component standard names to collide. The error must + show both colliding access paths so the user can spot the issue + immediately (this is the scm_type_defs / GFS_interstitial_type + sliced-view-vs-array pattern).""" + # parent_ddt has two sibling fields of the same inner_ddt type: + # one bare and one with a slice in the local name. Both + # flatten into entries for ``foo_std`` (inner_ddt's only + # component) — the second insertion triggers the duplicate. + src = ''' +[ccpp-table-properties] + name = inner_ddt + type = ddt +[ccpp-arg-table] + name = inner_ddt + type = ddt +[ foo ] + standard_name = foo_std + units = 1 + dimensions = () + type = integer + +[ccpp-table-properties] + name = parent_ddt + type = ddt +[ccpp-arg-table] + name = parent_ddt + type = ddt +[ inner_a ] + standard_name = inner_ddt_instance_a + units = ddt + dimensions = () + type = inner_ddt +[ inner_b ] + standard_name = inner_ddt_instance_b + units = ddt + dimensions = () + type = inner_ddt + +[ccpp-table-properties] + name = my_host + type = host +[ccpp-arg-table] + name = my_host + type = host +[ parent ] + standard_name = parent_ddt_instance + units = ddt + dimensions = () + type = parent_ddt +''' + tables = _parse_lines(src.splitlines(keepends=True), 'm.meta') + ddt_tables = [t for t in tables if t.table_type == 'ddt'] + host_tables = [t for t in tables if t.table_type == 'host'] + with self.assertRaises(CCPPError) as cm: + build_flat_host_dict(host_tables, [], ddt_tables) + msg = str(cm.exception) + # The duplicated standard name appears. + self.assertIn('foo_std', msg) + # Both colliding access paths appear so the user can diagnose. + self.assertIn('parent%inner_a%foo', msg) + self.assertIn('parent%inner_b%foo', msg) + # Hint about sibling-DDT-instance pattern is present. + self.assertIn('sibling DDT instances', msg) + + def test_missing_ddt_table_raises(self): + """DDT instance without corresponding DDT table → CCPPError.""" + host_tables = _parse_file('host_with_ddt_instance.meta') + with self.assertRaises(CCPPError) as cm: + build_flat_host_dict(host_tables, [], []) + self.assertIn('gfs_statein_type', str(cm.exception)) + + def test_host_character_assumed_length_raises(self): + """A host character variable with kind=len=* is rejected: host + metadata must give a concrete length (len=* is only valid for a + scheme dummy argument).""" + src = ''' +[ccpp-table-properties] + name = host_data + type = host +[ccpp-arg-table] + name = host_data + type = host +[ scheme_name ] + standard_name = scheme_name + units = none + dimensions = () + type = character + kind = len=* +''' + tables = _parse_lines(src.splitlines(keepends=True), 'h.meta') + with self.assertRaises(CCPPError) as cm: + build_flat_host_dict(tables, [], []) + msg = str(cm.exception) + self.assertIn('scheme_name', msg) + self.assertIn('len=*', msg) + + def test_control_character_assumed_length_ok(self): + """Control-table character variables are EXEMPT: they are pass-through + dummy arguments (suite_name, errmsg, ...) that the generated caps + declare ``character(len=*)``, so len=* is valid there.""" + src = ''' +[ccpp-table-properties] + name = ccpp_control + type = control +[ccpp-arg-table] + name = ccpp_control + type = control +[ label ] + standard_name = some_label + units = none + dimensions = () + type = character + kind = len=* +''' + tables = _parse_lines(src.splitlines(keepends=True), 'c.meta') + d = build_flat_host_dict([], tables, []) + self.assertEqual(d['some_label'].kind, 'len=*') + + def test_host_character_concrete_length_ok(self): + """A concrete host character length is accepted unchanged.""" + src = ''' +[ccpp-table-properties] + name = host_data + type = host +[ccpp-arg-table] + name = host_data + type = host +[ scheme_name ] + standard_name = scheme_name + units = none + dimensions = () + type = character + kind = len=512 +''' + tables = _parse_lines(src.splitlines(keepends=True), 'h.meta') + d = build_flat_host_dict(tables, [], []) + self.assertEqual(d['scheme_name'].kind, 'len=512') + + def test_host_vars_not_control(self): + # Host vars are is_control=False; loop bounds are now control vars (control table). + host_tables = _parse_file('host_simple.meta') + d = build_flat_host_dict(host_tables, [], []) + self.assertFalse(d['horizontal_dimension'].is_control) + self.assertFalse(d['vertical_layer_dimension'].is_control) + # Control vars from the control table should be is_control=True. + ctrl_tables = _parse_file('control_simple.meta') + dc = build_flat_host_dict([], ctrl_tables, []) + self.assertTrue(dc['horizontal_loop_begin'].is_control) + self.assertTrue(dc['horizontal_loop_end'].is_control) + + def test_nested_ddt_full_expansion(self): + outer_tables = _parse_file('ddt_nested_outer.meta') + inner_tables = _parse_file('ddt_nested_inner.meta') + host_tables = _parse_file('host_with_nested_ddt.meta') + d = build_flat_host_dict(host_tables, [], outer_tables + inner_tables) + self.assertIn('outer_ddt_instance', d) + self.assertIn('outer_scalar_field', d) + self.assertIn('inner_real_value', d) + self.assertIn('inner_integer_flag', d) + self.assertEqual(d['inner_real_value'].access_path, + 'outer_inst%inner_ddt%inner_value') + + def test_empty_inputs(self): + d = build_flat_host_dict([], [], []) + self.assertEqual(d, {}) + + +######################################################################## +# Tests: SchemeStore +######################################################################## + +_SIMPLE_SCHEME_SRC = '''\ +[ccpp-table-properties] + name = my_scheme + type = scheme + +[ccpp-arg-table] + name = my_scheme_run + type = scheme +[ im ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + intent = in +[ temp ] + standard_name = air_temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout + +[ccpp-arg-table] + name = my_scheme_init + type = scheme +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + + +class TestRule2LeafScalarDimRejection(unittest.TestCase): + """Rule 2 of the registered-scalar-index-dimension contract (see + capgen/metadata/registered_dimensions.py): leaf data variables — + intrinsic-typed or external-typed, the kind a scheme binds to — + MUST NOT declare a registered scalar-index dim like + ``number_of_threads``. ``build_flat_host_dict`` is the validation + site; the error must name the variable, the offending dim, the + paired index, and point at the registered_dimensions module. + """ + + _HOST_SRC = ''' +[ccpp-table-properties] + name = host_data + type = host + +[ccpp-arg-table] + name = host_data + type = host +[ leaf_field ] + standard_name = bad_leaf + long_name = leaf with a registered scalar-index dim — illegal + units = K + dimensions = (number_of_threads, horizontal_dimension) + type = real + kind = kind_phys +''' + + def test_leaf_with_registered_dim_rejected(self): + host_tbls = _parse_lines(self._HOST_SRC.splitlines(keepends=True), + 'host_bad.meta') + with self.assertRaises(CCPPError) as ctx: + build_flat_host_dict(host_tbls, [], []) + msg = str(ctx.exception) + # Names the offending variable. + self.assertIn("'leaf_field'", msg) + # Names the offending dim. + self.assertIn("'number_of_threads'", msg) + # Names the paired index. + self.assertIn("'thread_number'", msg) + # Points at the source module for further reading. + self.assertIn('registered_dimensions.py', msg) + # Tells the user how to fix it. + self.assertIn('container DDT', msg) + + def test_leaf_with_instances_dim_rejected(self): + src = self._HOST_SRC.replace( + 'number_of_threads, horizontal_dimension', + 'number_of_instances, horizontal_dimension', + ) + host_tbls = _parse_lines(src.splitlines(keepends=True), + 'host_bad.meta') + with self.assertRaises(CCPPError) as ctx: + build_flat_host_dict(host_tbls, [], []) + msg = str(ctx.exception) + self.assertIn("'number_of_instances'", msg) + self.assertIn("'instance_number'", msg) + + def test_ddt_instance_with_non_registered_dim_skips_field_flatten(self): + """A DDT-instance variable dimensioned by a non-registered dim + (e.g. ``horizontal_dimension`` on a per-column DDT array like + ``fluxLW(horizontal_dimension)`` of type ``ty_rad_lw``) is a + legitimate pattern: schemes take the whole sliced DDT array as + a single arg, not individual flattened inner fields. + + capgen must NOT flatten the inner fields in this case — + attempting to bake a scalar subscript would emit invalid + Fortran like ``parent%var%field(...)``. Instead, only the + DDT-instance's own entry is recorded; schemes that take it + whole resolve via that entry, and schemes that ask for inner + fields by std_name trip the existing "not found" error. + + Regression for the nested_suite + var_compat end-to-end fixtures + which use exactly this pattern. + """ + ddt_src = ''' +[ccpp-table-properties] + name = ty_rad_lw + type = ddt +[ccpp-arg-table] + name = ty_rad_lw + type = ddt +[ sfc_up_lw ] + standard_name = surface_upwelling_longwave_radiation_flux + units = W m-2 + dimensions = () + type = real + kind = kind_phys +''' + host_src = ''' +[ccpp-table-properties] + name = phys_state + type = host +[ccpp-arg-table] + name = phys_state + type = host +[ fluxLW ] + standard_name = longwave_radiation_fluxes + units = W m-2 + dimensions = (horizontal_dimension) + type = ty_rad_lw +''' + ddt_tbls = _parse_lines(ddt_src.splitlines(keepends=True), + 'module_rad_ddt.meta') + host_tbls = _parse_lines(host_src.splitlines(keepends=True), + 'phys_state.meta') + # No exception — the DDT-instance entry alone is enough for + # schemes that take the whole sliced DDT as an arg. + d = build_flat_host_dict(host_tbls, [], ddt_tbls) + self.assertIn('longwave_radiation_fluxes', d) + # Inner field is NOT flattened (would have required a scalar + # subscript capgen can't synthesize). + self.assertNotIn('surface_upwelling_longwave_radiation_flux', d) + + def test_ddt_instance_with_non_registered_dim_no_fields_accepted(self): + """An empty DDT (no fields) dimensioned by a non-registered dim + should NOT trigger the flatten-time error — there's nothing to + flatten, so no broken access pattern is possible. Real-world + case: ``ccpp_constituent_prop_ptr_t(:)`` field on a host's + constituent object. This DDT is accessed via the dedicated + constituent resolver, not via field-flattening.""" + ddt_src = ''' +[ccpp-table-properties] + name = empty_ddt + type = ddt +[ccpp-arg-table] + name = empty_ddt + type = ddt +''' + host_src = ''' +[ccpp-table-properties] + name = my_host + type = host +[ccpp-arg-table] + name = my_host + type = host +[ payload_arr ] + standard_name = some_payload_array + units = DDT + dimensions = (number_of_ccpp_constituents) + type = empty_ddt +''' + ddt_tbls = _parse_lines(ddt_src.splitlines(keepends=True), + 'empty_ddt.meta') + host_tbls = _parse_lines(host_src.splitlines(keepends=True), + 'my_host.meta') + # Should not raise. + result = build_flat_host_dict(host_tbls, [], ddt_tbls) + self.assertIn('some_payload_array', result) + + def test_container_ddt_with_registered_dim_accepted(self): + """The same dim on a DDT-instance container variable is fine — + Rule 2 only applies to leaves.""" + src = ''' +[ccpp-table-properties] + name = my_ddt + type = ddt + +[ccpp-arg-table] + name = my_ddt + type = ddt +[ field ] + standard_name = inner_field + units = K + dimensions = (horizontal_dimension) + type = real + kind = kind_phys +''' + host_src = ''' +[ccpp-table-properties] + name = host_data + type = host + +[ccpp-arg-table] + name = host_data + type = host +[ inst_array ] + standard_name = instance_array + units = DDT + dimensions = (number_of_threads) + type = my_ddt +''' + ddt_tbls = _parse_lines(src.splitlines(keepends=True), 'ddt.meta') + host_tbls = _parse_lines(host_src.splitlines(keepends=True), + 'host.meta') + # Should not raise — the dim is on a container. + result = build_flat_host_dict(host_tbls, [], ddt_tbls) + self.assertIn('inner_field', result) + # The flattened field's access path carries the (thread_number) + # placeholder. + self.assertEqual( + result['inner_field'].access_path, + 'inst_array(thread_number)%field', + ) + + +class TestSchemeStore(unittest.TestCase): + + def _build(self, src=_SIMPLE_SCHEME_SRC): + tables = _parse_lines(src.splitlines(keepends=True), 's.meta') + return SchemeStore.build_from(tables) + + def test_build_from_single_scheme(self): + store = self._build() + self.assertTrue(store.has_scheme('my_scheme')) + + def test_has_scheme_false(self): + store = self._build() + self.assertFalse(store.has_scheme('nonexistent')) + + def test_phases_for(self): + store = self._build() + self.assertEqual(sorted(store.phases_for('my_scheme')), ['init', 'run']) + + def test_phases_for_unknown(self): + store = self._build() + self.assertEqual(store.phases_for('nonexistent'), []) + + def test_variables_for_run(self): + store = self._build() + vars_ = store.variables_for('my_scheme', 'run') + self.assertIsNotNone(vars_) + std_names = [v.standard_name for v in vars_] + self.assertEqual(std_names, ['horizontal_dimension', 'air_temperature']) + + def test_variables_for_init(self): + store = self._build() + vars_ = store.variables_for('my_scheme', 'init') + self.assertIsNotNone(vars_) + std_names = [v.standard_name for v in vars_] + self.assertIn('ccpp_error_message', std_names) + self.assertIn('ccpp_error_code', std_names) + + def test_variables_for_absent_phase(self): + store = self._build() + self.assertIsNone(store.variables_for('my_scheme', 'final')) + + def test_variables_for_unknown_scheme(self): + store = self._build() + self.assertIsNone(store.variables_for('unknown', 'run')) + + def test_module_for_defaults_to_scheme_name(self): + store = self._build() + self.assertEqual(store.module_for('my_scheme'), 'my_scheme') + + def test_module_for_honors_table_property(self): + """``module_name`` in ``[ccpp-table-properties]`` overrides the + default (scheme-name) module.""" + src = ''' +[ccpp-table-properties] + name = effr_pre + type = scheme + module_name = mod_effr_pre + +[ccpp-arg-table] + name = effr_pre_run + type = scheme +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + tables = _parse_lines(src.splitlines(keepends=True), 's.meta') + store = SchemeStore.build_from(tables) + self.assertEqual(store.module_for('effr_pre'), 'mod_effr_pre') + + def test_module_for_unknown_scheme_returns_name(self): + store = self._build() + self.assertEqual(store.module_for('unknown_scheme'), 'unknown_scheme') + + def test_scheme_names_sorted(self): + src2 = _SIMPLE_SCHEME_SRC + ''' +[ccpp-table-properties] + name = another_scheme + type = scheme + +[ccpp-arg-table] + name = another_scheme_run + type = scheme +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + tables = _parse_lines(src2.splitlines(keepends=True), 's.meta') + store = SchemeStore.build_from(tables) + self.assertEqual(store.scheme_names(), ['another_scheme', 'my_scheme']) + + def test_non_scheme_tables_skipped(self): + host_src = '''\ +[ccpp-table-properties] + name = host_mod + type = host + +[ccpp-arg-table] + name = host_mod + type = host +[ im ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +''' + tables = (_parse_lines(_SIMPLE_SCHEME_SRC.splitlines(keepends=True), 's.meta') + + _parse_lines(host_src.splitlines(keepends=True), 'h.meta')) + store = SchemeStore.build_from(tables) + self.assertFalse(store.has_scheme('host_mod')) + self.assertTrue(store.has_scheme('my_scheme')) + + def test_duplicate_phase_raises(self): + src_dup = _SIMPLE_SCHEME_SRC + '''\ +[ccpp-arg-table] + name = my_scheme_run + type = scheme +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + tables = _parse_lines(src_dup.splitlines(keepends=True), 's.meta') + with self.assertRaises(CCPPError) as cm: + SchemeStore.build_from(tables) + self.assertIn('run', str(cm.exception)) + self.assertIn('my_scheme', str(cm.exception)) + + def test_duplicate_phase_names_both_source_files(self): + """When two distinct ``.meta`` files declare the same + (scheme, phase) pair, the error must name both file paths so + the user can locate the conflict instead of grepping the + ``--scheme-files`` list.""" + tables_a = _parse_lines( + _SIMPLE_SCHEME_SRC.splitlines(keepends=True), + '/projA/my_scheme.meta', + ) + tables_b = _parse_lines( + _SIMPLE_SCHEME_SRC.splitlines(keepends=True), + '/projB/my_scheme.meta', + ) + with self.assertRaises(CCPPError) as cm: + SchemeStore.build_from(tables_a + tables_b) + msg = str(cm.exception) + self.assertIn('my_scheme', msg) + # Both paths appear, in order: original then duplicate. + idx_a = msg.find('/projA/my_scheme.meta') + idx_b = msg.find('/projB/my_scheme.meta') + self.assertGreaterEqual(idx_a, 0, 'original path not in error: ' + msg) + self.assertGreaterEqual(idx_b, 0, 'duplicate path not in error: ' + msg) + self.assertLess(idx_a, idx_b, + 'expected original (A) before duplicate (B)') + # Different paths → no CMake-list hint. + self.assertNotIn('--scheme-files', msg) + + def test_duplicate_phase_same_path_hints_at_cmake_list(self): + """The motivating SCM case: a single ``.meta`` path listed + twice in the host's ``--scheme-files`` argument (typically a + stray CMake list entry). Both reported paths are byte-equal, + and the message appends an explicit ``--scheme-files`` hint so + the user knows to look in the build glue, not in the + metadata.""" + path = '/host/ccpp/physics/GWD/ugwpv1_gsldrag.meta' + tables_first = _parse_lines( + _SIMPLE_SCHEME_SRC.splitlines(keepends=True), path, + ) + tables_again = _parse_lines( + _SIMPLE_SCHEME_SRC.splitlines(keepends=True), path, + ) + with self.assertRaises(CCPPError) as cm: + SchemeStore.build_from(tables_first + tables_again) + msg = str(cm.exception) + # The single path appears (at least once; same string both + # places, so a single substring search suffices). + self.assertIn(path, msg) + # CMake-list duplication hint fires when the paths are equal. + self.assertIn('--scheme-files', msg) + self.assertIn('identical', msg) + + def test_build_from_scheme_files(self): + """Integration: build SchemeStore from the multipart scheme sample file.""" + from metadata.metadata_table import parse_metadata_file + tables = parse_metadata_file(_sample('scheme_multipart.meta')) + store = SchemeStore.build_from(tables) + self.assertTrue(store.has_scheme('temp_calc_adjust')) + self.assertEqual(sorted(store.phases_for('temp_calc_adjust')), + ['final', 'init', 'run']) + + def test_repr(self): + store = self._build() + self.assertIn('my_scheme', repr(store)) + + def test_variable_list_is_copy(self): + """Mutating the returned list must not affect the store's internal state.""" + store = self._build() + vars1 = store.variables_for('my_scheme', 'run') + vars1.append(None) + vars2 = store.variables_for('my_scheme', 'run') + self.assertEqual(len(vars2), 2) + + +######################################################################## +# Doctest loader +######################################################################## + +def load_tests(loader, tests, ignore): + """Auto-discover doctests from variable_resolver and related modules.""" + import metadata.variable_resolver as vr + tests.addTests(doctest.DocTestSuite(vr)) + return tests + + +######################################################################## +# main +######################################################################## + +if __name__ == '__main__': + unittest.main(verbosity=2)