Added -DBits cmake option for selective build, w/ auto-trimming of built backend interfaces

BUILD_OPTIONS.md

@@ -17,6 +17,15 @@ For a more complete list of cmake variables, take a look in the file `CMakeCache
 # Ditch GAMBIT components that you don't intend to use: itch
 -Ditch="ColliderBit;NeutrinoBit;Mathematica"
 
+# Select only the Bits you need: Bits
+# Similar to -Ditch, but opt-in rather than opt-out. If some Bits 
+# are specified using -DBits, all other Bits except ScannerBit are 
+# added to the ditch list. -DBits and -Ditch must not overlap.
+-DBits="DarkBit;PrecisionBit;SpecBit;DecayBit"          # typical dark matter project
+-DBits="ColliderBit;PrecisionBit;SpecBit;DecayBit"      # typical collider project
+-DBits="CosmoBit;DarkBit"                               # typical cosmology project
+
+
 # List the FlexibleSUSY models to build: BUILD_FS_MODELS
 # The names of the available FlexibleSUSY models correspond to 
 # the subdirectories in 
@@ -87,5 +96,16 @@ For a more complete list of cmake variables, take a look in the file `CMakeCache
 # Create Graphviz files: HAVE_GRAPHVIZ (On|Off)
 -DHAVE_GRAPHVIZ=On
 
+
+# Skip the build of any backend interface that no enabled Bit references
+# at the source level: GAMBIT_TRIM_BACKEND_INTERFACES (On|Off)
+# Auto-set to ON when -DBits is used or when -Ditch removes any Bit.
+# Set this to OFF to disable the trimming explicitly, e.g. when developing 
+# a new backend that no Bit yet references. Pair with 
+# -DGAMBIT_FORCE_BACKEND_INTERFACE="Name1;Name2" to keep specific backends 
+# regardless.
+-DGAMBIT_TRIM_BACKEND_INTERFACES=On
+-DGAMBIT_FORCE_BACKEND_INTERFACE="Acropolis;DarkCast"
+
 ```
 

Backends/scripts/backend_harvester.py

@@ -66,34 +66,63 @@ def main(argv):
     backend_type_headers = set([])
     bossed_backend_type_headers = set([])
     exclude_backends=set([])
+    enabled_bits=set([])
+    force_keep_backends=set([])
 
     # Handle command line options
     verbose = False
+    verbose_build = False    
     try:
-        opts, args = getopt.getopt(argv,"vx:",["verbose","exclude-backends="])
+        opts, args = getopt.getopt(argv,"vx:",
+            ["verbose","exclude-backends=","bits=","force-backends=","verbose-build"])
     except getopt.GetoptError:
         print('Usage: backend_harvestor.py [flags]')
         print(' flags:')
         print('        -v                       : More verbose output')
         print('        -x backend1,backend2,... : Exclude backend1, backend2, etc.')
+        print('        --bits=B1,B2,...         : Restrict the build to dependencies of these Bits.')
+        print('        --force-backends=A,B,... : Keep these backends, even if no enabled Bit needs them.')
+        print('        --verbose-build          : Print which frontends were auto-excluded.')
         sys.exit(2)
     for opt, arg in opts:
       if opt in ('-v','--verbose'):
         verbose = True
         print('backend_harvester.py: verbose=True')
       elif opt in ('-x','--exclude-backends'):
         exclude_backends.update(neatsplit(",",arg))
-
-    # Get list of frontend header files to include in backend_rollcall.hpp
-    frontend_headers.update(retrieve_generic_headers(verbose,"./Backends/include/gambit/Backends/frontends","frontend",exclude_backends))
-    frontend_headers_excluded.update(retrieve_generic_headers(verbose,"./Backends/include/gambit/Backends/frontends","frontend",exclude_backends, retrieve_excluded=True))
-    # Get list of backend type header files
+      elif opt == '--bits':
+        enabled_bits.update(b for b in neatsplit(",",arg) if b)
+      elif opt == '--force-backends':
+        force_keep_backends.update(b for b in neatsplit(",",arg) if b)
+      elif opt == '--verbose-build':
+        verbose_build = True
+
+    # If --bits is given, drop frontends that no enabled Bit references.
+    auto_excluded = set()
+    if enabled_bits:
+        auto_excluded, used_backends, all_backends = derive_optin_backend_excludes(
+            enabled_bits, ".",
+            "./Backends/include/gambit/Backends/frontends",
+            force_keep_backends)
+        exclude_backends.update(auto_excluded)
+        if verbose_build or verbose:
+            print("backend_harvester.py: enabled Bits = {0}".format(sorted(enabled_bits)))
+            if force_keep_backends:
+                print("backend_harvester.py: force-kept backends = {0}".format(sorted(force_keep_backends)))
+            print("backend_harvester.py: auto-excluded {0} backend(s) (no enabled Bit references them):".format(len(auto_excluded)))
+            for be in sorted(auto_excluded):
+                print("  - {0}".format(be))
+
+    # Discover backend type headers up front. Backends with BOSSed type
+    # headers must remain available so their type definitions stay in the
+    # rollcall, even if the user (or the auto-trim) tried to exclude them.
     backend_type_headers.update(retrieve_generic_headers(verbose,"./Backends/include/gambit/Backends/backend_types","backend type",set([])))
     bossed_backend_type_headers.update(retrieve_generic_headers(verbose,"./Backends/include/gambit/Backends/backend_types","BOSSed type",set([])))
-    # Remove bossed backends from list of excluded backends
     exclude_backends = set([be for be in exclude_backends if not any([excluded(bossed_be, [be]) for bossed_be in bossed_backend_type_headers])])
-    # Get list of frontend header files to include in backend_rollcall.hpp
+
+    # Partition the frontend headers using the final exclude set.
     frontend_headers.update(retrieve_generic_headers(verbose,"./Backends/include/gambit/Backends/frontends","frontend",exclude_backends))
+    frontend_headers_excluded.update(retrieve_generic_headers(verbose,"./Backends/include/gambit/Backends/frontends","frontend",exclude_backends, retrieve_excluded=True))
 
     if verbose:
         print("Frontend headers identified:")
@@ -210,7 +239,7 @@ def main(argv):
         print("Generated backend_types_rollcall.hpp.\n")
 
     import yaml
-    with open("./config/gambit_backends.yaml", "w+") as f:
+    with open("./config/gambit_backend_interfaces.yaml", "w+") as f:
         yaml.dump({
             "enabled": extract_yaml_for_diagnostic(frontend_headers),
             "disabled": extract_yaml_for_diagnostic(frontend_headers_excluded)

Backends/scripts/list_backends.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+"""Print a status table (one row per canonical BACKENDNAME) for every known
+GAMBIT backend interface. Reads config/gambit_backend_interfaces.yaml plus
+a data file written by CMakeLists.txt at configure time."""
+import io
+import os
+import sys
+
+import yaml
+
+_HERE = os.path.dirname(os.path.abspath(__file__))
+_REPO = os.path.normpath(os.path.join(_HERE, "..", ".."))
+sys.path.insert(0, os.path.join(_REPO, "Utils", "scripts"))
+
+from harvesting_tools import (  # noqa: E402
+    compute_bits_per_backend,
+    get_all_backendnames,
+)
+import list_table  # noqa: E402
+
+
+def main():
+    if len(sys.argv) != 2:
+        sys.stderr.write("Usage: list_backends.py <data_file>\n")
+        sys.exit(2)
+
+    active_targets = []
+    build_dir = ""
+    ditched_set = set()
+    with io.open(sys.argv[1], "r") as f:
+        for line in f:
+            if line.startswith("TARGETS="):
+                active_targets = [s for s in line[len("TARGETS="):].rstrip("\n").split(";") if s]
+            elif line.startswith("BUILD_DIR="):
+                build_dir = line[len("BUILD_DIR="):].rstrip("\n").strip()
+            elif line.startswith("DITCHED="):
+                ditched_set = {s for s in line[len("DITCHED="):].rstrip("\n").split(";") if s}
+
+    def is_ditched(canonical):
+        # Match canonical (or its underscore-collapsed form, for cases like
+        # SUSY_HIT vs susyhit_1.5) against any cmake ${itch} entry, with or
+        # without a trailing _<ver>. Catches BOSSed backends that the
+        # interfaces YAML keeps "enabled" for type availability after the
+        # build itself was excluded.
+        cl = canonical.lower()
+        cl_collapsed = canonical.replace('_', '').lower()
+        for entry in ditched_set:
+            e = entry.lower()
+            if e == cl or e.startswith(cl + "_"):
+                return True
+            if cl_collapsed != cl and (e == cl_collapsed or e.startswith(cl_collapsed + "_")):
+                return True
+        return False
+
+    def is_target_installed(target):
+        if not build_dir or not target:
+            return False
+        return os.path.exists(os.path.join(
+            build_dir, target + "-prefix", "src",
+            target + "-stamp", target + "-done"))
+
+    with io.open(os.path.join(_REPO, "config", "gambit_backend_interfaces.yaml"), "r") as f:
+        bdata = yaml.safe_load(f) or {}
+    enabled_set = set((bdata.get("enabled") or {}).keys())
+
+    frontend_dir = os.path.join(_REPO, "Backends", "include", "gambit", "Backends", "frontends")
+    all_backends = get_all_backendnames(frontend_dir)
+    bits_per = compute_bits_per_backend(_REPO, frontend_dir)
+
+    # Group each target under its longest-matching canonical (so e.g.
+    # darksusy_MSSM_6.4.0 lands under DarkSUSY_MSSM, not under DarkSUSY).
+    canonicals_by_len = sorted(all_backends, key=len, reverse=True)
+    targets_by_canonical = {b: [] for b in all_backends}
+    for t in active_targets:
+        tl = t.lower()
+        matched = False
+        for canonical in canonicals_by_len:
+            if tl.startswith(canonical.lower() + "_"):
+                targets_by_canonical[canonical].append(t)
+                matched = True
+                break
+        # Fallback for canonicals whose make target drops internal underscores
+        # (SUSY_HIT -> susyhit_1.5).
+        if not matched:
+            for canonical in canonicals_by_len:
+                cl = canonical.replace("_", "").lower()
+                if cl != canonical.lower() and tl.startswith(cl + "_"):
+                    targets_by_canonical[canonical].append(t)
+                    break
+
+    rows = []
+    for canonical in sorted(all_backends, key=lambda s: s.lower()):
+        targets = sorted(targets_by_canonical[canonical])
+        active = (canonical in enabled_set) and not is_ditched(canonical)
+        bits = bits_per.get(canonical, [])
+        rows.append((canonical, active, bits, targets))
+
+    name_w = max(len(r[0]) for r in rows)
+    name_w = max(name_w, len("Backend"))
+    bits_w = max(len(", ".join(r[2]) if r[2] else "none") for r in rows)
+    # "used by: " column spans the literal prefix (9 chars) plus the padded
+    # bits string. Bump to fit the "Used by Bits" header if narrower.
+    used_by_col_w = max(9 + bits_w, len("Used by Bits"))
+    bits_w = used_by_col_w - 9
+    print("  {bold}{h1:<{nw}}  {h2:<{tw}}  {h3:<{w3}}  {h4}{reset}".format(
+        bold=list_table.BOLD, reset=list_table.RESET,
+        h1="Backend", nw=name_w,
+        h2="Status", tw=list_table.TAG_W,
+        h3="Used by Bits", w3=used_by_col_w,
+        h4="Make targets"))
+    print("  {0}  {1}  {2}  {3}".format(
+        "-" * name_w, "-" * list_table.TAG_W,
+        "-" * used_by_col_w, "-" * len("Make targets")))
+
+    for name, active, bits, targets in rows:
+        if bits:
+            bits_visible = ", ".join(bits)
+            bits_field   = bits_visible + " " * (bits_w - len(bits_visible))
+        else:
+            bits_field = list_table.DIM + "none" + list_table.RESET + " " * (bits_w - len("none"))
+
+        if not targets:
+            targets_field = list_table.DIM + "none" + list_table.RESET
+        else:
+            ann = []
+            for t in targets:
+                if is_target_installed(t):
+                    ann.append("{} [installed]".format(t))
+                else:
+                    ann.append(t)
+            targets_field = ", ".join(ann)
+
+        if not active:
+            kind = "disabled"
+        elif targets and any(is_target_installed(t) for t in targets):
+            kind = "installed"
+        elif targets:
+            kind = "not_installed"
+        else:
+            kind = ""
+        print("  {name:<{nw}}  {tag}  used by: {bits}  targets: {tgts}".format(
+            name=name, nw=name_w, tag=list_table.tag_for_kind(kind),
+            bits=bits_field, tgts=targets_field))
+
+
+if __name__ == "__main__":
+    main()

Backends/src/frontends/simple_xs_1_0.cpp

@@ -16,7 +16,7 @@
 ///  *********************************************
 
 #include "gambit/Backends/frontend_macros.hpp"
-#include "gambit/Backends/frontends/simplexs_1_0.hpp"
+#include "gambit/Backends/frontends/simple_xs_1_0.hpp"
 
 BE_INI_FUNCTION {}
 END_BE_INI_FUNCTION

CMakeLists.txt

@@ -553,7 +553,9 @@ endif()
 add_custom_target(docs WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} COMMAND doxygen doc/doxygen.conf)
 
 # Work out which modules to include in the compile
+gambit_translate_bits_into_itch()
 retrieve_bits(GAMBIT_BITS ${PROJECT_SOURCE_DIR} "${itch}" "Loud")
+gambit_configure_optin_build()
 
 # Set up targets to make standalone tarballs of the different modules
 add_standalone_tarballs("${GAMBIT_BITS}" "${GAMBIT_VERSION_FULL}")
@@ -629,6 +631,14 @@ list(REMOVE_ITEM MODULE_HARVESTER_FILES "${PROJECT_SOURCE_DIR}/ScannerBit//inclu
 list(APPEND MODULE_HARVESTER_FILES "${PROJECT_SOURCE_DIR}/config/resolution_type_equivalency_classes.yaml")
 set(MODULE_HARVESTER_FILES ${MODULE_HARVESTER_FILES} ${BACKEND_HARVESTER_FILES})
 remove_build_files(models_harvested backends_harvested modules_harvested printers_harvested colliders_harvested)
+
+# Add backends and scanners. Done before the harvest registration so late
+# additions to ${itch} (e.g. the FeynHiggs gfortran>=10 ditch) reach the
+# harvester.
+include(cmake/externals.cmake)
+
+string (REPLACE ";" "," itch_with_commas "${itch}")
+
 if(EXISTS "${PROJECT_SOURCE_DIR}/Elements/")
   add_gambit_custom(module_harvest modules_harvested MODULE_HARVESTER MODULE_HARVESTER_FILES ${itch_with_commas})
 endif()
@@ -652,14 +662,11 @@ endif()
 
 # Generate the CMakeLists.txt files for GAMBIT modules, Backends, Models and Printers)
 message("${Yellow}-- Updating GAMBIT module, model, backend, and printer CMake files.${ColourReset}")
-set(update_cmakelists ${PROJECT_SOURCE_DIR}/cmake/scripts/update_cmakelists.py -x __not_a_real_name__,${itch_with_commas})
+set(update_cmakelists ${PROJECT_SOURCE_DIR}/cmake/scripts/update_cmakelists.py -x __not_a_real_name__,${itch_with_commas} ${update_cmakelists_extra_args})
 execute_process(RESULT_VARIABLE result COMMAND ${Python3_EXECUTABLE} ${update_cmakelists} WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
 check_result(${result} ${update_cmakelists})
 message("${Yellow}-- Updating GAMBIT module, backend, and printer CMake files - done.${ColourReset}")
 
-# Add backends and scanners
-include(cmake/externals.cmake)
-
 # Add GAMBIT subdirectories.
 add_subdirectory(Logs)
 add_subdirectory(Utils)
@@ -699,3 +706,23 @@ include(cmake/executables.cmake)
 if(EXISTS "${PROJECT_SOURCE_DIR}/ScannerBit/")
   include(${PROJECT_BINARY_DIR}/linkedout.cmake)
 endif()
+
+# `make list-backends` and `make list-scanners` status-table targets.
+# Each writes its data file on every cmake configure.
+file(WRITE "${PROJECT_BINARY_DIR}/list_backends_data.txt"
+     "TARGETS=${GAMBIT_AVAILABLE_BACKEND_TARGETS}\nBUILD_DIR=${PROJECT_BINARY_DIR}\nDITCHED=${itch}\n")
+add_custom_target(list-backends
+  COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/Backends/scripts/list_backends.py ${PROJECT_BINARY_DIR}/list_backends_data.txt
+  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+  USES_TERMINAL)
+add_dependencies(list-backends backend_harvest)
+
+file(WRITE "${PROJECT_BINARY_DIR}/list_scanners_data.txt"
+     "EXTERNAL_TARGETS=${GAMBIT_AVAILABLE_SCANNER_TARGETS}\nPYTHON_SCANNERS=${GAMBIT_PYTHON_SCANNER_STATUS}\nBUILD_DIR=${PROJECT_BINARY_DIR}\n")
+add_custom_target(list-scanners
+  COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/ScannerBit/scripts/list_scanners.py ${PROJECT_BINARY_DIR}/list_scanners_data.txt
+  WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+  USES_TERMINAL)
+
+message("${BoldYellow}-- Run `make list-backends` to see the available backend make targets.${ColourReset}")
+message("${BoldYellow}-- Run `make list-scanners` to see the available scanner make targets.${ColourReset}")

Core/src/diagnostics.cpp

@@ -90,9 +90,9 @@ namespace Gambit
   void gambit_core::backend_diagnostic()
   {
 
-    YAML::Node gambit_backends_yaml = YAML::LoadFile(GAMBIT_DIR "/config/gambit_backends.yaml");
-    auto gambit_backends = gambit_backends_yaml["enabled"].as<std::map<std::string, std::vector<std::string>>>();
-    auto gambit_backends_disabled = gambit_backends_yaml["disabled"].as<std::map<std::string, std::vector<std::string>>>();
+    YAML::Node backend_interfaces_yaml = YAML::LoadFile(GAMBIT_DIR "/config/gambit_backend_interfaces.yaml");
+    auto gambit_backends = backend_interfaces_yaml["enabled"].as<std::map<std::string, std::vector<std::string>>>();
+    auto gambit_backends_disabled = backend_interfaces_yaml["disabled"].as<std::map<std::string, std::vector<std::string>>>();
 
     for (auto &backend : gambit_backends_disabled)
     {