From a608e21c4a3ad728e1ce336d73671109bedff337 Mon Sep 17 00:00:00 2001 From: Anders Kvellestad Date: Sat, 9 May 2026 00:47:23 +0200 Subject: [PATCH 01/12] Added -DBits cmake option for selecting which Bits to build, w/ auto-trimming of backend interfaces. - Added new CMake option for the user to select which Bits to include in the build. (ScannerBit is always included.) Example: "cmake -DBits=ColliderBit;DecayBit .." will ditch all Bits except ColliderBit and DecayBit. - Also added system for automatic filtering of the backend interfaces (frontends), so that we only build the frontends for the backends that are actually used by the Bits we include in our build. - Added a new make target "make list-backends" which lists all the available backends, their make targets, and what Bit(s) they are relevant for - In our cmake system, replaced some embedded Python code with native cmake code for significant speed increase when checking the ditch status of build elements. --- BUILD_OPTIONS.md | 19 ++ .../{simplexs_1_0.hpp => simple_xs_1_0.hpp} | 0 Backends/scripts/backend_harvester.py | 45 +++- Backends/scripts/list_backends.py | 84 +++++++ Backends/src/frontends/simple_xs_1_0.cpp | 2 +- CMakeLists.txt | 18 +- Utils/scripts/harvesting_tools.py | 114 ++++++++++ cmake/backends.cmake | 4 +- cmake/cleaning.cmake | 5 +- cmake/externals.cmake | 18 +- cmake/scripts/update_cmakelists.py | 25 ++- cmake/utilities.cmake | 208 +++++++++++++++++- 12 files changed, 521 insertions(+), 21 deletions(-) rename Backends/include/gambit/Backends/frontends/{simplexs_1_0.hpp => simple_xs_1_0.hpp} (100%) create mode 100644 Backends/scripts/list_backends.py diff --git a/BUILD_OPTIONS.md b/BUILD_OPTIONS.md index 88fe52e6fb..11e298fe18 100644 --- a/BUILD_OPTIONS.md +++ b/BUILD_OPTIONS.md @@ -17,6 +17,25 @@ For a more complete list of cmake variables, take a look in the file `CMakeCache # Ditch GAMBIT components that you don't intend to use: itch -Ditch="ColliderBit;NeutrinoBit;Mathematica" +# Select only the Bits you need: Bits +# ScannerBit is always kept implicitly. Combines with -Ditch: +# anything explicitly ditched stays ditched even if listed in -DBits. +-DBits="DarkBit;PrecisionBit;SpecBit;DecayBit" # typical dark matter project +-DBits="ColliderBit;PrecisionBit;SpecBit;DecayBit" # typical collider project +-DBits="CosmoBit;DarkBit" # typical cosmology project + + +# Skip the build of any backend interface that no enabled Bit references +# at the source level: GAMBIT_TRIM_BACKEND_INTERFACES (On|Off) +# Auto-set to ON when -DBits is used or when -Ditch removes any Bit; forced ON +# below to opt in even when keeping every Bit. Set to OFF to disable the +# trimming explicitly (e.g. when developing a new backend that no Bit yet +# references). Pair with -DGAMBIT_FORCE_BACKEND_INTERFACE="Name1;Name2" to +# keep specific backends regardless. +-DGAMBIT_TRIM_BACKEND_INTERFACES=On +-DGAMBIT_FORCE_BACKEND_INTERFACE="Acropolis;DarkCast" + + # List the FlexibleSUSY models to build: BUILD_FS_MODELS # The names of the available FlexibleSUSY models correspond to # the subdirectories in diff --git a/Backends/include/gambit/Backends/frontends/simplexs_1_0.hpp b/Backends/include/gambit/Backends/frontends/simple_xs_1_0.hpp similarity index 100% rename from Backends/include/gambit/Backends/frontends/simplexs_1_0.hpp rename to Backends/include/gambit/Backends/frontends/simple_xs_1_0.hpp diff --git a/Backends/scripts/backend_harvester.py b/Backends/scripts/backend_harvester.py index 01afbdc0f6..f46d9bc22c 100644 --- a/Backends/scripts/backend_harvester.py +++ b/Backends/scripts/backend_harvester.py @@ -66,16 +66,23 @@ def main(argv): backend_type_headers = set([]) bossed_backend_type_headers = set([]) exclude_backends=set([]) + enabled_bits=set([]) + force_keep_backends=set([]) # Handle command line options verbose = False + verbose_build = False try: - opts, args = getopt.getopt(argv,"vx:",["verbose","exclude-backends="]) + opts, args = getopt.getopt(argv,"vx:", + ["verbose","exclude-backends=","bits=","force-backends=","verbose-build"]) except getopt.GetoptError: print('Usage: backend_harvestor.py [flags]') print(' flags:') print(' -v : More verbose output') print(' -x backend1,backend2,... : Exclude backend1, backend2, etc.') + print(' --bits=B1,B2,... : Restrict the build to dependencies of these Bits.') + print(' --force-backends=A,B,... : Keep these backends, even if no enabled Bit needs them.') + print(' --verbose-build : Print which frontends were auto-excluded.') sys.exit(2) for opt, arg in opts: if opt in ('-v','--verbose'): @@ -83,17 +90,39 @@ def main(argv): print('backend_harvester.py: verbose=True') elif opt in ('-x','--exclude-backends'): exclude_backends.update(neatsplit(",",arg)) - - # Get list of frontend header files to include in backend_rollcall.hpp - frontend_headers.update(retrieve_generic_headers(verbose,"./Backends/include/gambit/Backends/frontends","frontend",exclude_backends)) - frontend_headers_excluded.update(retrieve_generic_headers(verbose,"./Backends/include/gambit/Backends/frontends","frontend",exclude_backends, retrieve_excluded=True)) - # Get list of backend type header files + elif opt == '--bits': + enabled_bits.update(b for b in neatsplit(",",arg) if b) + elif opt == '--force-backends': + force_keep_backends.update(b for b in neatsplit(",",arg) if b) + elif opt == '--verbose-build': + verbose_build = True + + # If --bits is given, drop frontends that no enabled Bit references. + auto_excluded = set() + if enabled_bits: + auto_excluded, used_backends, all_backends = derive_optin_backend_excludes( + enabled_bits, ".", + "./Backends/include/gambit/Backends/frontends", + force_keep_backends) + exclude_backends.update(auto_excluded) + if verbose_build or verbose: + print("backend_harvester.py: enabled Bits = {0}".format(sorted(enabled_bits))) + if force_keep_backends: + print("backend_harvester.py: force-kept backends = {0}".format(sorted(force_keep_backends))) + print("backend_harvester.py: auto-excluded {0} backend(s) (no enabled Bit references them):".format(len(auto_excluded))) + for be in sorted(auto_excluded): + print(" - {0}".format(be)) + + # Discover backend type headers up front. Backends with BOSSed type + # headers must remain available so their type definitions stay in the + # rollcall, even if the user (or the auto-trim) tried to exclude them. backend_type_headers.update(retrieve_generic_headers(verbose,"./Backends/include/gambit/Backends/backend_types","backend type",set([]))) bossed_backend_type_headers.update(retrieve_generic_headers(verbose,"./Backends/include/gambit/Backends/backend_types","BOSSed type",set([]))) - # Remove bossed backends from list of excluded backends exclude_backends = set([be for be in exclude_backends if not any([excluded(bossed_be, [be]) for bossed_be in bossed_backend_type_headers])]) - # Get list of frontend header files to include in backend_rollcall.hpp + + # Partition the frontend headers using the final exclude set. frontend_headers.update(retrieve_generic_headers(verbose,"./Backends/include/gambit/Backends/frontends","frontend",exclude_backends)) + frontend_headers_excluded.update(retrieve_generic_headers(verbose,"./Backends/include/gambit/Backends/frontends","frontend",exclude_backends, retrieve_excluded=True)) if verbose: print("Frontend headers identified:") diff --git a/Backends/scripts/list_backends.py b/Backends/scripts/list_backends.py new file mode 100644 index 0000000000..0f9553a6bb --- /dev/null +++ b/Backends/scripts/list_backends.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Print a status table for every known GAMBIT backend interface. +Reads config/gambit_backends.yaml (regenerated by backend_harvester.py) for +the enabled/disabled interface status, and the data file written by +CMakeLists.txt at configure time (TARGETS=) for the set of +active backend make targets. Walks every Bit's include/src/examples tree +to find which Bits reference each backend. Prints one row per canonical +BACKENDNAME with its status, a "used by:" list of Bits, and the +backend's user-invokable make targets. +""" +from __future__ import print_function + +import io +import os +import sys + +import yaml + +_HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.normpath(os.path.join(_HERE, "..", "..", "Utils", "scripts"))) + +from harvesting_tools import ( # noqa: E402 + compute_bits_per_backend, + get_all_backendnames, +) + + +def main(): + if len(sys.argv) != 2: + sys.stderr.write("Usage: list_backends.py \n") + sys.exit(2) + + active_targets = [] + with io.open(sys.argv[1], "r") as f: + for line in f: + if line.startswith("TARGETS="): + active_targets = [s for s in line[len("TARGETS="):].rstrip("\n").split(";") if s] + break + + with io.open("./config/gambit_backends.yaml", "r") as f: + bdata = yaml.safe_load(f) or {} + enabled_set = set((bdata.get("enabled") or {}).keys()) + + frontend_dir = "./Backends/include/gambit/Backends/frontends" + all_backends = get_all_backendnames(frontend_dir) + bits_per = compute_bits_per_backend(".", frontend_dir) + + # Group each active target under its longest-matching canonical (so e.g. + # darksusy_MSSM_6.4.0 lands under DarkSUSY_MSSM, not under DarkSUSY). + canonicals_by_len = sorted(all_backends, key=len, reverse=True) + targets_by_canonical = {b: [] for b in all_backends} + for t in active_targets: + tl = t.lower() + for canonical in canonicals_by_len: + if tl.startswith(canonical.lower() + "_"): + targets_by_canonical[canonical].append(t) + break + + rows = [] + for canonical in sorted(all_backends, key=lambda s: s.lower()): + targets = sorted(targets_by_canonical[canonical]) + active = canonical in enabled_set + bits = bits_per.get(canonical, []) + rows.append((canonical, active, bits, targets)) + + name_w = max(len(r[0]) for r in rows) + bits_w = max(len(", ".join(r[2]) if r[2] else "(none)") for r in rows) + use_color = sys.stdout.isatty() + YELLOW = "\033[33m" if use_color else "" + RESET = "\033[0m" if use_color else "" + + for name, active, bits, targets in rows: + bits_str = ", ".join(bits) if bits else "(none)" + targets_str = ", ".join(targets) if targets else "(none)" + if active: + tag = " " * 10 # blank slot, same width as [disabled] + else: + tag = YELLOW + "[disabled]" + RESET + print(" {name:<{nw}} {tag} used by: {bits:<{bw}} targets: {tgts}".format( + name=name, nw=name_w, tag=tag, bits=bits_str, bw=bits_w, tgts=targets_str)) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/Backends/src/frontends/simple_xs_1_0.cpp b/Backends/src/frontends/simple_xs_1_0.cpp index 290b6b48c3..7d7f6d80c8 100644 --- a/Backends/src/frontends/simple_xs_1_0.cpp +++ b/Backends/src/frontends/simple_xs_1_0.cpp @@ -16,7 +16,7 @@ /// ********************************************* #include "gambit/Backends/frontend_macros.hpp" -#include "gambit/Backends/frontends/simplexs_1_0.hpp" +#include "gambit/Backends/frontends/simple_xs_1_0.hpp" BE_INI_FUNCTION {} END_BE_INI_FUNCTION diff --git a/CMakeLists.txt b/CMakeLists.txt index 91767f2481..ffcb8faabd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -553,7 +553,9 @@ endif() add_custom_target(docs WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} COMMAND doxygen doc/doxygen.conf) # Work out which modules to include in the compile +gambit_translate_bits_into_itch() retrieve_bits(GAMBIT_BITS ${PROJECT_SOURCE_DIR} "${itch}" "Loud") +gambit_configure_optin_build() # Set up targets to make standalone tarballs of the different modules add_standalone_tarballs("${GAMBIT_BITS}" "${GAMBIT_VERSION_FULL}") @@ -652,7 +654,7 @@ endif() # Generate the CMakeLists.txt files for GAMBIT modules, Backends, Models and Printers) message("${Yellow}-- Updating GAMBIT module, model, backend, and printer CMake files.${ColourReset}") -set(update_cmakelists ${PROJECT_SOURCE_DIR}/cmake/scripts/update_cmakelists.py -x __not_a_real_name__,${itch_with_commas}) +set(update_cmakelists ${PROJECT_SOURCE_DIR}/cmake/scripts/update_cmakelists.py -x __not_a_real_name__,${itch_with_commas} ${update_cmakelists_extra_args}) execute_process(RESULT_VARIABLE result COMMAND ${Python3_EXECUTABLE} ${update_cmakelists} WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) check_result(${result} ${update_cmakelists}) message("${Yellow}-- Updating GAMBIT module, backend, and printer CMake files - done.${ColourReset}") @@ -699,3 +701,17 @@ include(cmake/executables.cmake) if(EXISTS "${PROJECT_SOURCE_DIR}/ScannerBit/") include(${PROJECT_BINARY_DIR}/linkedout.cmake) endif() + +# Print the list of backend make targets that survived the ditch / opt-in pass. +gambit_print_available_backend_targets() + +# Register a `make list-backends` target that prints a status table of every +# known backend, including which Bits use it. The data file is rewritten on +# each cmake configure to reflect the current set of active targets. +file(WRITE "${PROJECT_BINARY_DIR}/list_backends_data.txt" + "TARGETS=${GAMBIT_AVAILABLE_BACKEND_TARGETS}\n") +add_custom_target(list-backends + COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/Backends/scripts/list_backends.py ${PROJECT_BINARY_DIR}/list_backends_data.txt + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + USES_TERMINAL) +add_dependencies(list-backends backend_harvest) diff --git a/Utils/scripts/harvesting_tools.py b/Utils/scripts/harvesting_tools.py index 18df7dcee0..8bb0f2888a 100644 --- a/Utils/scripts/harvesting_tools.py +++ b/Utils/scripts/harvesting_tools.py @@ -686,6 +686,120 @@ def retrieve_generic_headers(verbose, starting_dir, kind, excludes, exclude_list return headers +# Helpers for the opt-in build path (-Dbits / GAMBIT_AUTO_TRIM_BACKENDS). +_SOURCE_EXTS = (".cpp", ".cc", ".c", ".hpp", ".hh", ".h") + +def derive_backendname_from_filename(stem): + """Derive a backend name from a frontend filename stem by dropping version-number tokens (e.g. DarkSUSY_MSSM_6_4_0 -> DarkSUSY_MSSM).""" + return "_".join(p for p in stem.split("_") if p and not p[0].isdigit()) + + +def get_all_backendnames(frontend_dir): + """Return the set of distinct BACKENDNAMEs declared by frontend headers, with a filename fallback for BOSSed frontends.""" + backends = set() + if not os.path.isdir(frontend_dir): + return backends + define_re = re.compile(r'^\s*#\s*define\s+BACKENDNAME\s+(\S+)', re.MULTILINE) + for fn in os.listdir(frontend_dir): + if not fn.endswith(".hpp"): continue + path = os.path.join(frontend_dir, fn) + try: + with io.open(path, "r", encoding="utf-8", errors="replace") as f: + text = f.read() + except (IOError, OSError): + continue + m = define_re.search(text) + if m: + backends.add(m.group(1)) + else: + backends.add(derive_backendname_from_filename(fn[:-len(".hpp")])) + return backends + + +def derive_used_backends_via_token_sweep(enabled_bits, project_root, all_backends): + """Return the subset of all_backends whose name occurs as a token in any enabled Bit's include/src/examples tree.""" + if not all_backends or not enabled_bits: + return set() + # Sort longest-first so the alternation prefers e.g. DarkSUSY_MSSM over DarkSUSY. + sorted_backends = sorted(all_backends, key=len, reverse=True) + # Match BACKENDNAME at a token boundary, but allow a trailing _suffix so + # that e.g. `BEreq::SUSYHD_MHiggs(...)` counts as a SUSYHD reference. + pattern = re.compile( + r'(? Date: Sat, 9 May 2026 09:42:12 +0200 Subject: [PATCH 02/12] First version of a new "make list-scanners" target for our build system. Similar in spirit to the "make list-backends" target. --- CMakeLists.txt | 12 ++ ScannerBit/scripts/list_scanners.py | 217 ++++++++++++++++++++++++++++ cmake/externals.cmake | 5 + cmake/python_scanners.cmake | 8 + 4 files changed, 242 insertions(+) create mode 100644 ScannerBit/scripts/list_scanners.py diff --git a/CMakeLists.txt b/CMakeLists.txt index ffcb8faabd..850862bc25 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -715,3 +715,15 @@ add_custom_target(list-backends WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} USES_TERMINAL) add_dependencies(list-backends backend_harvest) + +# Register a `make list-scanners` target that prints an analogous status table +# for scanner plugins (external, native, and Python). The data file is +# rewritten on each cmake configure; the script also reads +# scratch/build_time/scanbit_excluded_libs.yaml (written by ScannerBit) and +# walks ScannerBit/src/scanners/ for the canonical native plugin list. +file(WRITE "${PROJECT_BINARY_DIR}/list_scanners_data.txt" + "EXTERNAL_TARGETS=${GAMBIT_AVAILABLE_SCANNER_TARGETS}\nPYTHON_SCANNERS=${GAMBIT_PYTHON_SCANNER_STATUS}\n") +add_custom_target(list-scanners + COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/ScannerBit/scripts/list_scanners.py ${PROJECT_BINARY_DIR}/list_scanners_data.txt + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + USES_TERMINAL) diff --git a/ScannerBit/scripts/list_scanners.py b/ScannerBit/scripts/list_scanners.py new file mode 100644 index 0000000000..6cec0661a1 --- /dev/null +++ b/ScannerBit/scripts/list_scanners.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Print a status table of every known GAMBIT scanner plugin. + +Combines three data sources: + * the data file written by CMakeLists.txt at configure time (active external + scanner make targets and per-Python-scanner status), passed in argv[1]; + * scratch/build_time/scanbit_excluded_libs.yaml (disabled native libs), + produced by ScannerBit/CMakeLists.txt; + * the scanner_plugin(name, version(...)) registrations under + ScannerBit/src/scanners/, which are the canonical list of native plugins. + +Prints one row per canonical plugin name. External plugins list their +versioned make targets; native-only plugins are tagged 'native'; Python +scanners are tagged 'python'. +""" +from __future__ import print_function + +import io +import os +import re +import sys + +import yaml + + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_REPO = os.path.normpath(os.path.join(_HERE, "..", "..")) + + +def parse_data_file(path): + external_targets = [] + python_scanners = [] + with io.open(path, "r") as f: + for line in f: + if line.startswith("EXTERNAL_TARGETS="): + external_targets = [s for s in line[len("EXTERNAL_TARGETS="):].rstrip("\n").split(";") if s] + elif line.startswith("PYTHON_SCANNERS="): + python_scanners = [s for s in line[len("PYTHON_SCANNERS="):].rstrip("\n").split(";") if s] + return external_targets, python_scanners + + +def parse_excluded_yaml(path): + """Return {libname: [reason, ...]} for every excluded library, where libname + is the bare library identifier (e.g. "scanner_great", "scanner_python").""" + excluded = {} + if not os.path.exists(path): + return excluded + with io.open(path, "r") as f: + data = yaml.safe_load(f) or {} + for libfile, info in data.items(): + # libfile looks like "libscanner_great.so" or "libobjective_python.so". + name = libfile + if name.startswith("lib"): + name = name[3:] + if name.endswith(".so"): + name = name[:-3] + raw_reasons = info.get("reason") or [] + if isinstance(raw_reasons, str): + raw_reasons = [raw_reasons] + # Each reason was emitted by ScannerBit/CMakeLists.txt as a literal + # YAML sequence entry like '- file missing: "ROOT"', which the YAML + # parser turns into a single-key mapping. Flatten back to "key: value". + reasons = [] + for r in raw_reasons: + if isinstance(r, dict): + for k, v in r.items(): + reasons.append("{}: {}".format(str(k).strip(), str(v).strip())) + elif r is not None: + s = str(r).strip() + if s: + reasons.append(s) + excluded[name] = reasons + return excluded + + +_PLUGIN_RE = re.compile( + r"scanner_plugin\s*\(\s*([A-Za-z_][A-Za-z0-9_]*)\s*,\s*version\s*\(\s*([^)]*)\)\s*\)" +) + + +def parse_native_plugins(scanners_dir): + """Walk ScannerBit/src/scanners//*.cpp and collect every + scanner_plugin(name, version(...)) registration. Returns a list of dicts: + { 'libname': 'scanner_', 'plugin': '', 'version': '' }. + + Skips the python/ directory; Python scanners come from a separate data + source (cmake/python_scanners.cmake).""" + rows = [] + if not os.path.isdir(scanners_dir): + return rows + for libdir in sorted(os.listdir(scanners_dir)): + libpath = os.path.join(scanners_dir, libdir) + if not os.path.isdir(libpath): + continue + if libdir == "python": + continue + libname = "scanner_" + libdir + for root, _dirs, files in os.walk(libpath): + for fname in files: + if not fname.endswith((".cpp", ".cc", ".cxx", ".hpp", ".h")): + continue + with io.open(os.path.join(root, fname), "r", errors="replace") as fh: + text = fh.read() + for m in _PLUGIN_RE.finditer(text): + name = m.group(1) + parts = [p.strip() for p in m.group(2).split(",") if p.strip()] + ver = ".".join(parts) + rows.append({"libname": libname, "plugin": name, "version": ver}) + return rows + + +def main(): + if len(sys.argv) != 2: + sys.stderr.write("Usage: list_scanners.py \n") + sys.exit(2) + + external_targets, python_scanner_lines = parse_data_file(sys.argv[1]) + excluded = parse_excluded_yaml( + os.path.join(_REPO, "scratch", "build_time", "scanbit_excluded_libs.yaml") + ) + native = parse_native_plugins(os.path.join(_REPO, "ScannerBit", "src", "scanners")) + + # Map external make target to its companion plugin library, e.g. + # "diver_1.3.0" -> "scanner_diver_1.3.0". Most ScannerBit/src/scanners/ + # subdirs include the version suffix, but a few (e.g. "great") do not, so + # also register a versionless fallback mapping ("scanner_great"). + target_for_lib = {} + for tgt in external_targets: + target_for_lib.setdefault("scanner_" + tgt, tgt) + if "_" in tgt: + target_for_lib.setdefault("scanner_" + tgt.rsplit("_", 1)[0], tgt) + + # Group native+external plugins by canonical plugin name. Each entry is a + # list of per-version dicts, sorted by version. + grouped = {} + for r in native: + libname = r["libname"] + target = target_for_lib.get(libname, "") + grouped.setdefault(r["plugin"], []).append({ + "version": r["version"], + "library": libname, + "target": target, + "disabled": libname in excluded, + "reason": "; ".join(excluded.get(libname, [])), + }) + + use_color = sys.stdout.isatty() + YELLOW = "\033[33m" if use_color else "" + DIM = "\033[2m" if use_color else "" + RESET = "\033[0m" if use_color else "" + + def disable_tag(disabled): + return (YELLOW + "[disabled]" + RESET) if disabled else " " * 10 + + # Build (name, row-disabled, info-string, reason-string) tuples for the + # grouped (native+external) section. + grouped_rows = [] + for name in sorted(grouped, key=str.lower): + versions = sorted(grouped[name], key=lambda v: v["version"]) + all_disabled = all(v["disabled"] for v in versions) + any_disabled = any(v["disabled"] for v in versions) + external_versions = [v for v in versions if v["target"]] + native_versions = [v for v in versions if not v["target"]] + + parts = [] + if external_versions: + tgt_strs = [] + for v in external_versions: + if v["disabled"] and not all_disabled: + tgt_strs.append("{} [disabled]".format(v["target"])) + else: + tgt_strs.append(v["target"]) + parts.append("targets: " + ", ".join(tgt_strs)) + if native_versions and not external_versions: + parts.append("native") + info = " ".join(parts) + + grouped_rows.append((name, all_disabled, any_disabled and not all_disabled, info)) + + # Python rows: one per scanner. Status is libscanner_python lib status (if + # excluded, all are disabled) overlaid with per-scanner Python module + # availability. + py_lib_disabled = "scanner_python" in excluded + python_rows = [] + for line in python_scanner_lines: + parts = line.split("|") + if len(parts) < 2: + continue + pname = parts[0] + pstatus = parts[1] + disabled = py_lib_disabled or (pstatus != "enabled") + python_rows.append((pname, disabled)) + + # Compute column widths across both sections so they line up. + all_names = [r[0] for r in grouped_rows] + [r[0] for r in python_rows] + if not all_names: + print(" (no scanner plugins found)") + return + name_w = max(len(n) for n in all_names) + + # Print grouped (native+external) rows. + for name, all_disabled, partial, info in grouped_rows: + if partial: + tag = DIM + "[partial] " + RESET + else: + tag = disable_tag(all_disabled) + print(" {n:<{nw}} {tag} {info}".format( + n=name, nw=name_w, tag=tag, info=info)) + + # Print Python rows (sorted alphabetically for parity with the grouped section). + for name, disabled in sorted(python_rows, key=lambda r: r[0].lower()): + print(" {n:<{nw}} {tag} python".format( + n=name, nw=name_w, tag=disable_tag(disabled))) + + +if __name__ == "__main__": + main() diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 74f108bf5a..c3e4c88e55 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -167,6 +167,11 @@ macro(add_extra_targets type package ver dir dl target) list(APPEND GAMBIT_AVAILABLE_BACKEND_TARGETS "${package}_${ver}") endif() + # Track available scanner make targets for the list-scanners helper. + if (${type} STREQUAL "scanner") + list(APPEND GAMBIT_AVAILABLE_SCANNER_TARGETS "${package}_${ver}") + endif() + #Add extra targets common to everything. enable_auto_rebuild(${pname}) set_target_properties(${pname} PROPERTIES EXCLUDE_FROM_ALL 1) diff --git a/cmake/python_scanners.cmake b/cmake/python_scanners.cmake index 055c5fbdf7..b0cfc0c300 100644 --- a/cmake/python_scanners.cmake +++ b/cmake/python_scanners.cmake @@ -53,6 +53,14 @@ macro(check_python_scanner_modules name modules packages) if(packages_missing_${name}) string (REPLACE "," " " missing "${packages_missing_${name}}") message(" To enable the scanner ${name}, please install the following Python packages:${missing}") + # Record the missing-package status for the list-scanners helper. Strip the + # leading comma left by the accumulator and re-encode "," as "+" to keep + # the per-scanner record self-contained inside our ";"-separated list. + string(REGEX REPLACE "^," "" _missing_csv "${packages_missing_${name}}") + string(REPLACE "," "+" _missing_csv "${_missing_csv}") + list(APPEND GAMBIT_PYTHON_SCANNER_STATUS "${name}|missing|${_missing_csv}") + else() + list(APPEND GAMBIT_PYTHON_SCANNER_STATUS "${name}|enabled|") endif() endmacro() From 8fef5b57984f9c698f77cfa974ea13c07d29594f Mon Sep 17 00:00:00 2001 From: Anders Kvellestad Date: Sat, 9 May 2026 09:54:01 +0200 Subject: [PATCH 03/12] Clarified column in the "make list-scanners" output. --- ScannerBit/scripts/list_scanners.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ScannerBit/scripts/list_scanners.py b/ScannerBit/scripts/list_scanners.py index 6cec0661a1..1f11c3c0fb 100644 --- a/ScannerBit/scripts/list_scanners.py +++ b/ScannerBit/scripts/list_scanners.py @@ -172,7 +172,7 @@ def disable_tag(disabled): tgt_strs.append(v["target"]) parts.append("targets: " + ", ".join(tgt_strs)) if native_versions and not external_versions: - parts.append("native") + parts.append("native (no make targets)") info = " ".join(parts) grouped_rows.append((name, all_disabled, any_disabled and not all_disabled, info)) @@ -209,7 +209,7 @@ def disable_tag(disabled): # Print Python rows (sorted alphabetically for parity with the grouped section). for name, disabled in sorted(python_rows, key=lambda r: r[0].lower()): - print(" {n:<{nw}} {tag} python".format( + print(" {n:<{nw}} {tag} python (no make targets)".format( n=name, nw=name_w, tag=disable_tag(disabled))) From 3fddf3522c17e60f5436447ff49653b61be9788d Mon Sep 17 00:00:00 2001 From: Anders Kvellestad Date: Sat, 9 May 2026 10:02:24 +0200 Subject: [PATCH 04/12] Add header with column names for "make list-backends" and "make list-scanners" --- Backends/scripts/list_backends.py | 16 ++++++++++++++++ ScannerBit/scripts/list_scanners.py | 12 +++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/Backends/scripts/list_backends.py b/Backends/scripts/list_backends.py index 0f9553a6bb..f601ea60d0 100644 --- a/Backends/scripts/list_backends.py +++ b/Backends/scripts/list_backends.py @@ -64,11 +64,27 @@ def main(): rows.append((canonical, active, bits, targets)) name_w = max(len(r[0]) for r in rows) + name_w = max(name_w, len("Name")) bits_w = max(len(", ".join(r[2]) if r[2] else "(none)") for r in rows) + # "used by: " column spans the literal prefix (9 chars) plus the padded + # bits string. Bump to fit the "Used by Bits" header if narrower. + used_by_col_w = max(9 + bits_w, len("Used by Bits")) + bits_w = used_by_col_w - 9 use_color = sys.stdout.isatty() YELLOW = "\033[33m" if use_color else "" + BOLD = "\033[1m" if use_color else "" RESET = "\033[0m" if use_color else "" + # Header row + dashed separator. + print(" {bold}{h1:<{nw}} {h2:<10} {h3:<{w3}} {h4}{reset}".format( + bold=BOLD, reset=RESET, + h1="Name", nw=name_w, + h2="Status", + h3="Used by Bits", w3=used_by_col_w, + h4="Make targets")) + print(" {0} {1} {2} {3}".format( + "-" * name_w, "-" * 10, "-" * used_by_col_w, "-" * len("Make targets"))) + for name, active, bits, targets in rows: bits_str = ", ".join(bits) if bits else "(none)" targets_str = ", ".join(targets) if targets else "(none)" diff --git a/ScannerBit/scripts/list_scanners.py b/ScannerBit/scripts/list_scanners.py index 1f11c3c0fb..b053d935ac 100644 --- a/ScannerBit/scripts/list_scanners.py +++ b/ScannerBit/scripts/list_scanners.py @@ -196,7 +196,17 @@ def disable_tag(disabled): if not all_names: print(" (no scanner plugins found)") return - name_w = max(len(n) for n in all_names) + name_w = max(max(len(n) for n in all_names), len("Name")) + + # Header row + dashed separator. + BOLD = "\033[1m" if use_color else "" + print(" {bold}{h1:<{nw}} {h2:<10} {h3}{reset}".format( + bold=BOLD, reset=RESET, + h1="Name", nw=name_w, + h2="Status", + h3="Make targets")) + print(" {0} {1} {2}".format( + "-" * name_w, "-" * 10, "-" * len("Make targets"))) # Print grouped (native+external) rows. for name, all_disabled, partial, info in grouped_rows: From 070a2f5efcfe12e354193611ff639b8bcc84f146 Mon Sep 17 00:00:00 2001 From: Anders Kvellestad Date: Sat, 9 May 2026 10:24:30 +0200 Subject: [PATCH 05/12] Added more status labels to the list-scanners and list-backends output. --- Backends/scripts/list_backends.py | 60 +++++++++++++++++--- CMakeLists.txt | 4 +- ScannerBit/scripts/list_scanners.py | 87 +++++++++++++++++++++++------ 3 files changed, 122 insertions(+), 29 deletions(-) diff --git a/Backends/scripts/list_backends.py b/Backends/scripts/list_backends.py index f601ea60d0..faa789091a 100644 --- a/Backends/scripts/list_backends.py +++ b/Backends/scripts/list_backends.py @@ -31,11 +31,23 @@ def main(): sys.exit(2) active_targets = [] + build_dir = "" with io.open(sys.argv[1], "r") as f: for line in f: if line.startswith("TARGETS="): active_targets = [s for s in line[len("TARGETS="):].rstrip("\n").split(";") if s] - break + elif line.startswith("BUILD_DIR="): + build_dir = line[len("BUILD_DIR="):].rstrip("\n").strip() + + def is_target_installed(target): + """ExternalProject_Add writes a -done stamp into its stamp dir + once configure/build/install all succeed; gambit's clean/nuke targets + remove it, so the stamp tracks reality.""" + if not build_dir or not target: + return False + return os.path.exists(os.path.join( + build_dir, target + "-prefix", "src", + target + "-stamp", target + "-done")) with io.open("./config/gambit_backends.yaml", "r") as f: bdata = yaml.safe_load(f) or {} @@ -72,26 +84,56 @@ def main(): bits_w = used_by_col_w - 9 use_color = sys.stdout.isatty() YELLOW = "\033[33m" if use_color else "" + GREEN = "\033[32m" if use_color else "" + CYAN = "\033[36m" if use_color else "" BOLD = "\033[1m" if use_color else "" RESET = "\033[0m" if use_color else "" + # Status column tags: width 15 to fit "[not installed]" (longest visible + # label). ANSI codes don't add visible width, so explicit space-padding + # after the colored bracket keeps every row aligned. + TAG_W = len("[not installed]") + def make_tag(label, color): + if not label: + return " " * TAG_W + return color + label + RESET + " " * (TAG_W - len(label)) + # Header row + dashed separator. - print(" {bold}{h1:<{nw}} {h2:<10} {h3:<{w3}} {h4}{reset}".format( + print(" {bold}{h1:<{nw}} {h2:<{tw}} {h3:<{w3}} {h4}{reset}".format( bold=BOLD, reset=RESET, h1="Name", nw=name_w, - h2="Status", + h2="Status", tw=TAG_W, h3="Used by Bits", w3=used_by_col_w, h4="Make targets")) print(" {0} {1} {2} {3}".format( - "-" * name_w, "-" * 10, "-" * used_by_col_w, "-" * len("Make targets"))) + "-" * name_w, "-" * TAG_W, "-" * used_by_col_w, "-" * len("Make targets"))) for name, active, bits, targets in rows: - bits_str = ", ".join(bits) if bits else "(none)" - targets_str = ", ".join(targets) if targets else "(none)" - if active: - tag = " " * 10 # blank slot, same width as [disabled] + bits_str = ", ".join(bits) if bits else "(none)" + if not targets: + targets_str = "(none)" + else: + ann = [] + for t in targets: + if is_target_installed(t): + ann.append("{} [installed]".format(t)) + else: + ann.append(t) + targets_str = ", ".join(ann) + + if not active: + kind = "disabled" + elif targets and any(is_target_installed(t) for t in targets): + kind = "installed" + elif targets: + kind = "not_installed" else: - tag = YELLOW + "[disabled]" + RESET + kind = "" + tag_color = {"disabled": YELLOW, "installed": GREEN, + "not_installed": CYAN, "": ""}[kind] + tag_label = {"disabled": "[disabled]", "installed": "[installed]", + "not_installed": "[not installed]", "": ""}[kind] + tag = make_tag(tag_label, tag_color) print(" {name:<{nw}} {tag} used by: {bits:<{bw}} targets: {tgts}".format( name=name, nw=name_w, tag=tag, bits=bits_str, bw=bits_w, tgts=targets_str)) diff --git a/CMakeLists.txt b/CMakeLists.txt index 850862bc25..322534e19c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -709,7 +709,7 @@ gambit_print_available_backend_targets() # known backend, including which Bits use it. The data file is rewritten on # each cmake configure to reflect the current set of active targets. file(WRITE "${PROJECT_BINARY_DIR}/list_backends_data.txt" - "TARGETS=${GAMBIT_AVAILABLE_BACKEND_TARGETS}\n") + "TARGETS=${GAMBIT_AVAILABLE_BACKEND_TARGETS}\nBUILD_DIR=${PROJECT_BINARY_DIR}\n") add_custom_target(list-backends COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/Backends/scripts/list_backends.py ${PROJECT_BINARY_DIR}/list_backends_data.txt WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} @@ -722,7 +722,7 @@ add_dependencies(list-backends backend_harvest) # scratch/build_time/scanbit_excluded_libs.yaml (written by ScannerBit) and # walks ScannerBit/src/scanners/ for the canonical native plugin list. file(WRITE "${PROJECT_BINARY_DIR}/list_scanners_data.txt" - "EXTERNAL_TARGETS=${GAMBIT_AVAILABLE_SCANNER_TARGETS}\nPYTHON_SCANNERS=${GAMBIT_PYTHON_SCANNER_STATUS}\n") + "EXTERNAL_TARGETS=${GAMBIT_AVAILABLE_SCANNER_TARGETS}\nPYTHON_SCANNERS=${GAMBIT_PYTHON_SCANNER_STATUS}\nBUILD_DIR=${PROJECT_BINARY_DIR}\n") add_custom_target(list-scanners COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/ScannerBit/scripts/list_scanners.py ${PROJECT_BINARY_DIR}/list_scanners_data.txt WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} diff --git a/ScannerBit/scripts/list_scanners.py b/ScannerBit/scripts/list_scanners.py index b053d935ac..55a27afa58 100644 --- a/ScannerBit/scripts/list_scanners.py +++ b/ScannerBit/scripts/list_scanners.py @@ -30,13 +30,28 @@ def parse_data_file(path): external_targets = [] python_scanners = [] + build_dir = "" with io.open(path, "r") as f: for line in f: if line.startswith("EXTERNAL_TARGETS="): external_targets = [s for s in line[len("EXTERNAL_TARGETS="):].rstrip("\n").split(";") if s] elif line.startswith("PYTHON_SCANNERS="): python_scanners = [s for s in line[len("PYTHON_SCANNERS="):].rstrip("\n").split(";") if s] - return external_targets, python_scanners + elif line.startswith("BUILD_DIR="): + build_dir = line[len("BUILD_DIR="):].rstrip("\n").strip() + return external_targets, python_scanners, build_dir + + +def is_target_installed(build_dir, target): + """An ExternalProject_Add target writes a -done stamp into its + stamp dir once configure/build/install all succeed. The stamp persists + across cmake reconfigures and is removed by the corresponding clean/nuke + targets, so it tracks reality.""" + if not build_dir or not target: + return False + stamp = os.path.join(build_dir, target + "-prefix", "src", + target + "-stamp", target + "-done") + return os.path.exists(stamp) def parse_excluded_yaml(path): @@ -114,7 +129,7 @@ def main(): sys.stderr.write("Usage: list_scanners.py \n") sys.exit(2) - external_targets, python_scanner_lines = parse_data_file(sys.argv[1]) + external_targets, python_scanner_lines, build_dir = parse_data_file(sys.argv[1]) excluded = parse_excluded_yaml( os.path.join(_REPO, "scratch", "build_time", "scanbit_excluded_libs.yaml") ) @@ -146,19 +161,29 @@ def main(): use_color = sys.stdout.isatty() YELLOW = "\033[33m" if use_color else "" + GREEN = "\033[32m" if use_color else "" + CYAN = "\033[36m" if use_color else "" DIM = "\033[2m" if use_color else "" RESET = "\033[0m" if use_color else "" - def disable_tag(disabled): - return (YELLOW + "[disabled]" + RESET) if disabled else " " * 10 - - # Build (name, row-disabled, info-string, reason-string) tuples for the - # grouped (native+external) section. + # Status column tags: width 15 to fit the longest visible label + # ("[not installed]"). ANSI color codes don't take visible width. + TAG_W = len("[not installed]") + def make_tag(label, color): + if not label: + return " " * TAG_W + return color + label + RESET + " " * (TAG_W - len(label)) + + # Build (name, status_kind, info-string) tuples for the grouped + # (native+external) section. status_kind is one of: + # "disabled" — every version is excluded + # "installed" — at least one external version's stamp exists + # "not_installed" — has external make targets but none built yet + # "" — native-only / nothing actionable to display grouped_rows = [] for name in sorted(grouped, key=str.lower): versions = sorted(grouped[name], key=lambda v: v["version"]) all_disabled = all(v["disabled"] for v in versions) - any_disabled = any(v["disabled"] for v in versions) external_versions = [v for v in versions if v["target"]] native_versions = [v for v in versions if not v["target"]] @@ -168,6 +193,8 @@ def disable_tag(disabled): for v in external_versions: if v["disabled"] and not all_disabled: tgt_strs.append("{} [disabled]".format(v["target"])) + elif is_target_installed(build_dir, v["target"]): + tgt_strs.append("{} [installed]".format(v["target"])) else: tgt_strs.append(v["target"]) parts.append("targets: " + ", ".join(tgt_strs)) @@ -175,7 +202,19 @@ def disable_tag(disabled): parts.append("native (no make targets)") info = " ".join(parts) - grouped_rows.append((name, all_disabled, any_disabled and not all_disabled, info)) + any_installed = any( + (not v["disabled"]) and is_target_installed(build_dir, v["target"]) + for v in external_versions + ) + if all_disabled: + kind = "disabled" + elif any_installed: + kind = "installed" + elif external_versions: + kind = "not_installed" + else: + kind = "" + grouped_rows.append((name, kind, info)) # Python rows: one per scanner. Status is libscanner_python lib status (if # excluded, all are disabled) overlaid with per-scanner Python module @@ -200,27 +239,39 @@ def disable_tag(disabled): # Header row + dashed separator. BOLD = "\033[1m" if use_color else "" - print(" {bold}{h1:<{nw}} {h2:<10} {h3}{reset}".format( + print(" {bold}{h1:<{nw}} {h2:<{tw}} {h3}{reset}".format( bold=BOLD, reset=RESET, h1="Name", nw=name_w, - h2="Status", + h2="Status", tw=TAG_W, h3="Make targets")) print(" {0} {1} {2}".format( - "-" * name_w, "-" * 10, "-" * len("Make targets"))) + "-" * name_w, "-" * TAG_W, "-" * len("Make targets"))) + + tag_color = { + "disabled": YELLOW, + "installed": GREEN, + "not_installed": CYAN, + "": "", + } + tag_label = { + "disabled": "[disabled]", + "installed": "[installed]", + "not_installed": "[not installed]", + "": "", + } # Print grouped (native+external) rows. - for name, all_disabled, partial, info in grouped_rows: - if partial: - tag = DIM + "[partial] " + RESET - else: - tag = disable_tag(all_disabled) + for name, kind, info in grouped_rows: + tag = make_tag(tag_label[kind], tag_color[kind]) print(" {n:<{nw}} {tag} {info}".format( n=name, nw=name_w, tag=tag, info=info)) # Print Python rows (sorted alphabetically for parity with the grouped section). for name, disabled in sorted(python_rows, key=lambda r: r[0].lower()): + kind = "disabled" if disabled else "" + tag = make_tag(tag_label[kind], tag_color[kind]) print(" {n:<{nw}} {tag} python (no make targets)".format( - n=name, nw=name_w, tag=disable_tag(disabled))) + n=name, nw=name_w, tag=tag)) if __name__ == "__main__": From 8bf6aafd9abd5d1de99fd61ed359462c56819cbf Mon Sep 17 00:00:00 2001 From: Anders Kvellestad Date: Sat, 9 May 2026 13:48:29 +0200 Subject: [PATCH 06/12] Improved scripts for determining at cmake time which Bits use which backends. --- Backends/scripts/list_backends.py | 37 +++-- ScannerBit/scripts/list_scanners.py | 24 +++- Utils/scripts/harvesting_tools.py | 207 ++++++++++++++++++++-------- 3 files changed, 192 insertions(+), 76 deletions(-) diff --git a/Backends/scripts/list_backends.py b/Backends/scripts/list_backends.py index faa789091a..97288ca1bd 100644 --- a/Backends/scripts/list_backends.py +++ b/Backends/scripts/list_backends.py @@ -63,10 +63,21 @@ def is_target_installed(target): targets_by_canonical = {b: [] for b in all_backends} for t in active_targets: tl = t.lower() + matched = False for canonical in canonicals_by_len: if tl.startswith(canonical.lower() + "_"): targets_by_canonical[canonical].append(t) + matched = True break + # Fallback: try the collapsed-underscore form, for canonicals where + # the make target drops internal underscores (e.g. SUSY_HIT + # -> susyhit_1.5). + if not matched: + for canonical in canonicals_by_len: + cl = canonical.replace("_", "").lower() + if cl != canonical.lower() and tl.startswith(cl + "_"): + targets_by_canonical[canonical].append(t) + break rows = [] for canonical in sorted(all_backends, key=lambda s: s.lower()): @@ -76,8 +87,8 @@ def is_target_installed(target): rows.append((canonical, active, bits, targets)) name_w = max(len(r[0]) for r in rows) - name_w = max(name_w, len("Name")) - bits_w = max(len(", ".join(r[2]) if r[2] else "(none)") for r in rows) + name_w = max(name_w, len("Backend")) + bits_w = max(len(", ".join(r[2]) if r[2] else "none") for r in rows) # "used by: " column spans the literal prefix (9 chars) plus the padded # bits string. Bump to fit the "Used by Bits" header if narrower. used_by_col_w = max(9 + bits_w, len("Used by Bits")) @@ -86,6 +97,7 @@ def is_target_installed(target): YELLOW = "\033[33m" if use_color else "" GREEN = "\033[32m" if use_color else "" CYAN = "\033[36m" if use_color else "" + DIM = "\033[2m" if use_color else "" BOLD = "\033[1m" if use_color else "" RESET = "\033[0m" if use_color else "" @@ -101,7 +113,7 @@ def make_tag(label, color): # Header row + dashed separator. print(" {bold}{h1:<{nw}} {h2:<{tw}} {h3:<{w3}} {h4}{reset}".format( bold=BOLD, reset=RESET, - h1="Name", nw=name_w, + h1="Backend", nw=name_w, h2="Status", tw=TAG_W, h3="Used by Bits", w3=used_by_col_w, h4="Make targets")) @@ -109,9 +121,18 @@ def make_tag(label, color): "-" * name_w, "-" * TAG_W, "-" * used_by_col_w, "-" * len("Make targets"))) for name, active, bits, targets in rows: - bits_str = ", ".join(bits) if bits else "(none)" + # Bits column: dim "none" when empty. ANSI codes don't take visible + # width, so do the column-padding manually rather than via {:<{bw}}. + if bits: + bits_visible = ", ".join(bits) + bits_field = bits_visible + " " * (bits_w - len(bits_visible)) + else: + bits_field = DIM + "none" + RESET + " " * (bits_w - len("none")) + + # Targets column: dim "none" when no targets; otherwise annotate + # individually-installed targets inline. if not targets: - targets_str = "(none)" + targets_field = DIM + "none" + RESET else: ann = [] for t in targets: @@ -119,7 +140,7 @@ def make_tag(label, color): ann.append("{} [installed]".format(t)) else: ann.append(t) - targets_str = ", ".join(ann) + targets_field = ", ".join(ann) if not active: kind = "disabled" @@ -134,8 +155,8 @@ def make_tag(label, color): tag_label = {"disabled": "[disabled]", "installed": "[installed]", "not_installed": "[not installed]", "": ""}[kind] tag = make_tag(tag_label, tag_color) - print(" {name:<{nw}} {tag} used by: {bits:<{bw}} targets: {tgts}".format( - name=name, nw=name_w, tag=tag, bits=bits_str, bw=bits_w, tgts=targets_str)) + print(" {name:<{nw}} {tag} used by: {bits} targets: {tgts}".format( + name=name, nw=name_w, tag=tag, bits=bits_field, tgts=targets_field)) if __name__ == "__main__": diff --git a/ScannerBit/scripts/list_scanners.py b/ScannerBit/scripts/list_scanners.py index 55a27afa58..b2a2cc7153 100644 --- a/ScannerBit/scripts/list_scanners.py +++ b/ScannerBit/scripts/list_scanners.py @@ -199,7 +199,7 @@ def make_tag(label, color): tgt_strs.append(v["target"]) parts.append("targets: " + ", ".join(tgt_strs)) if native_versions and not external_versions: - parts.append("native (no make targets)") + parts.append("targets: " + DIM + "none; native GAMBIT scanner" + RESET) info = " ".join(parts) any_installed = any( @@ -227,21 +227,23 @@ def make_tag(label, color): continue pname = parts[0] pstatus = parts[1] + # Restore "+" → ", " in the missing-pkgs field (see python_scanners.cmake). + pmissing = parts[2].replace("+", ", ") if len(parts) > 2 else "" disabled = py_lib_disabled or (pstatus != "enabled") - python_rows.append((pname, disabled)) + python_rows.append((pname, disabled, pmissing)) # Compute column widths across both sections so they line up. all_names = [r[0] for r in grouped_rows] + [r[0] for r in python_rows] if not all_names: print(" (no scanner plugins found)") return - name_w = max(max(len(n) for n in all_names), len("Name")) + name_w = max(max(len(n) for n in all_names), len("Scanner")) # Header row + dashed separator. BOLD = "\033[1m" if use_color else "" print(" {bold}{h1:<{nw}} {h2:<{tw}} {h3}{reset}".format( bold=BOLD, reset=RESET, - h1="Name", nw=name_w, + h1="Scanner", nw=name_w, h2="Status", tw=TAG_W, h3="Make targets")) print(" {0} {1} {2}".format( @@ -267,11 +269,19 @@ def make_tag(label, color): n=name, nw=name_w, tag=tag, info=info)) # Print Python rows (sorted alphabetically for parity with the grouped section). - for name, disabled in sorted(python_rows, key=lambda r: r[0].lower()): + for name, disabled, missing in sorted(python_rows, key=lambda r: r[0].lower()): kind = "disabled" if disabled else "" tag = make_tag(tag_label[kind], tag_color[kind]) - print(" {n:<{nw}} {tag} python (no make targets)".format( - n=name, nw=name_w, tag=tag)) + if disabled: + if missing: + body = "none; python plugin – install package(s) [{}] to enable".format(missing) + else: + body = "none; python plugin – install package(s) to enable" + else: + body = "none; python plugin" + info = "targets: " + DIM + body + RESET + print(" {n:<{nw}} {tag} {info}".format( + n=name, nw=name_w, tag=tag, info=info)) if __name__ == "__main__": diff --git a/Utils/scripts/harvesting_tools.py b/Utils/scripts/harvesting_tools.py index 8bb0f2888a..5ac4ff5bcb 100644 --- a/Utils/scripts/harvesting_tools.py +++ b/Utils/scripts/harvesting_tools.py @@ -716,43 +716,21 @@ def get_all_backendnames(frontend_dir): return backends -def derive_used_backends_via_token_sweep(enabled_bits, project_root, all_backends): - """Return the subset of all_backends whose name occurs as a token in any enabled Bit's include/src/examples tree.""" - if not all_backends or not enabled_bits: +def derive_used_backends(enabled_bits, project_root, frontend_dir): + """Return the set of backends used by any of the enabled Bits, derived + via rollcall-macro analysis (BACKEND_REQ / LONG_BACKEND_REQ capabilities + matched against frontend BE_FUNCTION/BE_VARIABLE/BE_CONV_FUNCTION + declarations, plus NEEDS_CLASSES_FROM for BOSSed-class deps, plus the + hard-coded overrides in _KNOWN_BIT_USES_OVERRIDES). + + This mirrors the resolution that the GAMBIT dependency resolver + performs at runtime, so it doesn't keep backends alive merely because + their name shows up in a doc comment or a string literal.""" + if not enabled_bits: return set() - # Sort longest-first so the alternation prefers e.g. DarkSUSY_MSSM over DarkSUSY. - sorted_backends = sorted(all_backends, key=len, reverse=True) - # Match BACKENDNAME at a token boundary, but allow a trailing _suffix so - # that e.g. `BEreq::SUSYHD_MHiggs(...)` counts as a SUSYHD reference. - pattern = re.compile( - r'(? 0: + if text[i] == '(': depth += 1 + elif text[i] == ')': depth -= 1 + i += 1 + quotes = re.findall(r'"([^"]+)"', text[m.end():i-1]) + if quotes: + caps.add(quotes[-1]) + return caps + +_BIT_REQ_RE = re.compile(r'\b(?:LONG_)?BACKEND_REQ(?:_FROM_GROUP)?\s*\(\s*([A-Za-z_][A-Za-z0-9_]*)') +_BIT_NCF_RE = re.compile(r'\bNEEDS_CLASSES_FROM\s*\(\s*([A-Za-z_][A-Za-z0-9_]*)') + + def compute_bits_per_backend(project_root, frontend_dir): - """Return a dict mapping each canonical BACKENDNAME to the sorted list of Bit names whose include/src/examples tree mentions it.""" + """Return a dict mapping each canonical BACKENDNAME to the sorted list of + Bit names that depend on it, using GAMBIT's rollcall macros as the + source of truth. + + A Bit is attributed to a backend if either: + * the Bit's rollcall headers contain (LONG_)BACKEND_REQ for any + capability the backend's frontend declares via BE_FUNCTION, + BE_VARIABLE, or BE_CONV_FUNCTION (following shared_includes), or + * the Bit's headers contain NEEDS_CLASSES_FROM(, ...), + used for BOSSed-class dependencies. + + A small hard-coded override map (_KNOWN_BIT_USES_OVERRIDES) covers the + cases where capability names are constructed via C-preprocessor token + pasting at the call site, or where the dependency is purely structural + (data files only, no rollcall-macro reference).""" all_backends = get_all_backendnames(frontend_dir) if not all_backends: return {} - sorted_backends = sorted(all_backends, key=len, reverse=True) - pattern = re.compile( - r'(?/Backends/include/gambit/Backends/frontends" + # 3 levels up gets us to "/Backends/include" — the include root + # used by the shared_includes/ path. + backends_inc_dir = os.path.normpath(os.path.join(frontend_dir, "..", "..", "..")) + caps_by_backend = {b: set() for b in all_backends} + if os.path.isdir(frontend_dir): + for fn in sorted(os.listdir(frontend_dir)): + if not fn.endswith(".hpp"): + continue + backend = derive_backendname_from_filename(fn[:-len(".hpp")]) + if backend not in all_backends: + continue + text = _read_frontend_recursive( + os.path.join(frontend_dir, fn), backends_inc_dir) + caps_by_backend[backend] |= _extract_be_capabilities(text) + + # Step 2: each Bit's requested capabilities and NEEDS_CLASSES_FROM names. bit_dirs = sorted(d for d in os.listdir(project_root) if 'Bit' in d and os.path.isdir(os.path.join(project_root, d))) + result = {b: set() for b in all_backends} for bit in bit_dirs: - bit_path = os.path.join(project_root, bit) - for sub in ("include", "src", "examples"): - walk_root = os.path.join(bit_path, sub) - if not os.path.isdir(walk_root): - continue - for root, _dirs, files in os.walk(walk_root): - for name in files: - if not name.endswith(_SOURCE_EXTS): - continue - try: - with io.open(os.path.join(root, name), "r", - encoding="utf-8", errors="replace") as f: - text = f.read() - except (IOError, OSError): - continue - for m in pattern.finditer(text): - result[m.group(1)].add(bit) + inc_dir = os.path.join(project_root, bit, "include") + if not os.path.isdir(inc_dir): + continue + bit_caps, bit_ncfs = set(), set() + for root, _dirs, files in os.walk(inc_dir): + for fn in files: + if not fn.endswith((".hpp", ".h", ".hh")): + continue + try: + with io.open(os.path.join(root, fn), "r", + encoding="utf-8", errors="replace") as f: + text = _strip_cxx_comments(f.read()) + except (IOError, OSError): + continue + for m in _BIT_REQ_RE.finditer(text): bit_caps.add(m.group(1)) + for m in _BIT_NCF_RE.finditer(text): bit_ncfs.add(m.group(1)) + for be in all_backends: + if (caps_by_backend[be] & bit_caps) or (be in bit_ncfs): + result[be].add(bit) + + # Step 3: apply hard-coded overrides. + for be, bits in _KNOWN_BIT_USES_OVERRIDES.items(): + if be in result: + result[be].update(bits) + return {b: sorted(s) for b, s in result.items()} From af932a89d361b44141de560ab52fe5925f87afae Mon Sep 17 00:00:00 2001 From: Anders Kvellestad Date: Sat, 9 May 2026 15:00:04 +0200 Subject: [PATCH 07/12] Clarified some confusion between activation/deactivation of backend interfaces (frontends) and ditching of backends. --- Backends/scripts/backend_harvester.py | 2 +- Backends/scripts/list_backends.py | 37 ++++++++++++++++++++++++--- CMakeLists.txt | 18 ++++++++++--- Core/src/diagnostics.cpp | 6 ++--- cmake/cleaning.cmake | 2 +- 5 files changed, 52 insertions(+), 13 deletions(-) diff --git a/Backends/scripts/backend_harvester.py b/Backends/scripts/backend_harvester.py index f46d9bc22c..7f37c7de3d 100644 --- a/Backends/scripts/backend_harvester.py +++ b/Backends/scripts/backend_harvester.py @@ -239,7 +239,7 @@ def main(argv): print("Generated backend_types_rollcall.hpp.\n") import yaml - with open("./config/gambit_backends.yaml", "w+") as f: + with open("./config/gambit_backend_interfaces.yaml", "w+") as f: yaml.dump({ "enabled": extract_yaml_for_diagnostic(frontend_headers), "disabled": extract_yaml_for_diagnostic(frontend_headers_excluded) diff --git a/Backends/scripts/list_backends.py b/Backends/scripts/list_backends.py index 97288ca1bd..09b54a80d8 100644 --- a/Backends/scripts/list_backends.py +++ b/Backends/scripts/list_backends.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Print a status table for every known GAMBIT backend interface. -Reads config/gambit_backends.yaml (regenerated by backend_harvester.py) for -the enabled/disabled interface status, and the data file written by +Reads config/gambit_backend_interfaces.yaml (regenerated by +backend_harvester.py) for the enabled/disabled interface status, and the data file written by CMakeLists.txt at configure time (TARGETS=) for the set of active backend make targets. Walks every Bit's include/src/examples tree to find which Bits reference each backend. Prints one row per canonical @@ -32,12 +32,35 @@ def main(): active_targets = [] build_dir = "" + ditched_set = set() with io.open(sys.argv[1], "r") as f: for line in f: if line.startswith("TARGETS="): active_targets = [s for s in line[len("TARGETS="):].rstrip("\n").split(";") if s] elif line.startswith("BUILD_DIR="): build_dir = line[len("BUILD_DIR="):].rstrip("\n").strip() + elif line.startswith("DITCHED="): + ditched_set = {s for s in line[len("DITCHED="):].rstrip("\n").split(";") if s} + + def is_ditched(canonical): + """Return True if any cmake ${itch} entry refers to this canonical + backend. Matches case-insensitively, accepts both bare names + (e.g. 'feynhiggs') and versioned names (e.g. 'feynhiggs_2.11.2'), + and tolerates the underscore-collapsed form (e.g. itch entry + 'susyhit' matching canonical 'SUSY_HIT'). The interfaces YAML may + still mark this backend "enabled" (BOSSed types must remain in the + rollcall even when the backend itself is excluded), but if cmake + has ditched it then there is no make target to build it and it + cannot actually be used in this configuration.""" + cl = canonical.lower() + cl_collapsed = canonical.replace('_', '').lower() + for entry in ditched_set: + e = entry.lower() + if e == cl or e.startswith(cl + "_"): + return True + if cl_collapsed != cl and (e == cl_collapsed or e.startswith(cl_collapsed + "_")): + return True + return False def is_target_installed(target): """ExternalProject_Add writes a -done stamp into its stamp dir @@ -49,7 +72,7 @@ def is_target_installed(target): build_dir, target + "-prefix", "src", target + "-stamp", target + "-done")) - with io.open("./config/gambit_backends.yaml", "r") as f: + with io.open("./config/gambit_backend_interfaces.yaml", "r") as f: bdata = yaml.safe_load(f) or {} enabled_set = set((bdata.get("enabled") or {}).keys()) @@ -82,7 +105,13 @@ def is_target_installed(target): rows = [] for canonical in sorted(all_backends, key=lambda s: s.lower()): targets = sorted(targets_by_canonical[canonical]) - active = canonical in enabled_set + # A backend is "active" iff its frontend interface compiles in (per + # the YAML's enabled list) AND cmake hasn't ditched the build path + # in the current configuration. The latter check catches cases like + # FeynHiggs (gfortran>=10) and BOSSed backends like Pythia/Rivet + # that the interfaces YAML keeps as "enabled" for type-availability + # reasons even after they've been auto-excluded. + active = (canonical in enabled_set) and not is_ditched(canonical) bits = bits_per.get(canonical, []) rows.append((canonical, active, bits, targets)) diff --git a/CMakeLists.txt b/CMakeLists.txt index 322534e19c..9e39f244c2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -631,6 +631,19 @@ list(REMOVE_ITEM MODULE_HARVESTER_FILES "${PROJECT_SOURCE_DIR}/ScannerBit//inclu list(APPEND MODULE_HARVESTER_FILES "${PROJECT_SOURCE_DIR}/config/resolution_type_equivalency_classes.yaml") set(MODULE_HARVESTER_FILES ${MODULE_HARVESTER_FILES} ${BACKEND_HARVESTER_FILES}) remove_build_files(models_harvested backends_harvested modules_harvested printers_harvested colliders_harvested) + +# Add backends and scanners. (Done before the harvest custom-target +# registration and update_cmakelists invocation below so that any late +# additions to ${itch} from cmake/backends.cmake / cmake/scanners.cmake — +# e.g. the FeynHiggs ditch for gfortran>=10 — are reflected in the +# itch_with_commas string we pass to those steps. Otherwise the backend +# harvester would still mark such backends "enabled" in +# gambit_backend_interfaces.yaml even though no make targets exist to build them.) +include(cmake/externals.cmake) + +# Reprocess the ditch set into a comma-separated list (final, post-externals). +string (REPLACE ";" "," itch_with_commas "${itch}") + if(EXISTS "${PROJECT_SOURCE_DIR}/Elements/") add_gambit_custom(module_harvest modules_harvested MODULE_HARVESTER MODULE_HARVESTER_FILES ${itch_with_commas}) endif() @@ -659,9 +672,6 @@ execute_process(RESULT_VARIABLE result COMMAND ${Python3_EXECUTABLE} ${update_cm check_result(${result} ${update_cmakelists}) message("${Yellow}-- Updating GAMBIT module, backend, and printer CMake files - done.${ColourReset}") -# Add backends and scanners -include(cmake/externals.cmake) - # Add GAMBIT subdirectories. add_subdirectory(Logs) add_subdirectory(Utils) @@ -709,7 +719,7 @@ gambit_print_available_backend_targets() # known backend, including which Bits use it. The data file is rewritten on # each cmake configure to reflect the current set of active targets. file(WRITE "${PROJECT_BINARY_DIR}/list_backends_data.txt" - "TARGETS=${GAMBIT_AVAILABLE_BACKEND_TARGETS}\nBUILD_DIR=${PROJECT_BINARY_DIR}\n") + "TARGETS=${GAMBIT_AVAILABLE_BACKEND_TARGETS}\nBUILD_DIR=${PROJECT_BINARY_DIR}\nDITCHED=${itch}\n") add_custom_target(list-backends COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/Backends/scripts/list_backends.py ${PROJECT_BINARY_DIR}/list_backends_data.txt WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} diff --git a/Core/src/diagnostics.cpp b/Core/src/diagnostics.cpp index bcb027f46f..b1c769465e 100644 --- a/Core/src/diagnostics.cpp +++ b/Core/src/diagnostics.cpp @@ -90,9 +90,9 @@ namespace Gambit void gambit_core::backend_diagnostic() { - YAML::Node gambit_backends_yaml = YAML::LoadFile(GAMBIT_DIR "/config/gambit_backends.yaml"); - auto gambit_backends = gambit_backends_yaml["enabled"].as>>(); - auto gambit_backends_disabled = gambit_backends_yaml["disabled"].as>>(); + YAML::Node backend_interfaces_yaml = YAML::LoadFile(GAMBIT_DIR "/config/gambit_backend_interfaces.yaml"); + auto gambit_backends = backend_interfaces_yaml["enabled"].as>>(); + auto gambit_backends_disabled = backend_interfaces_yaml["disabled"].as>>(); for (auto &backend : gambit_backends_disabled) { diff --git a/cmake/cleaning.cmake b/cmake/cleaning.cmake index 93a4eee335..0f9b80a249 100644 --- a/cmake/cleaning.cmake +++ b/cmake/cleaning.cmake @@ -73,7 +73,7 @@ if(EXISTS "${PROJECT_SOURCE_DIR}/ScannerBit/") endif() # Arrange for removal of other generated files upon "make clean". -set(clean_files ${clean_files} "${PROJECT_SOURCE_DIR}/config/gambit_backends.yaml") +set(clean_files ${clean_files} "${PROJECT_SOURCE_DIR}/config/gambit_backend_interfaces.yaml") # Add all the clean files set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${clean_files}") From 0f3bc991d860fd568e41b8152c13277d38e470a8 Mon Sep 17 00:00:00 2001 From: Anders Kvellestad Date: Sat, 9 May 2026 15:45:30 +0200 Subject: [PATCH 08/12] Further tweaks to list-scanners cmake system. Trying to clarify/unify status labels --- ScannerBit/scripts/list_scanners.py | 62 ++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/ScannerBit/scripts/list_scanners.py b/ScannerBit/scripts/list_scanners.py index b2a2cc7153..bedc05b232 100644 --- a/ScannerBit/scripts/list_scanners.py +++ b/ScannerBit/scripts/list_scanners.py @@ -54,6 +54,15 @@ def is_target_installed(build_dir, target): return os.path.exists(stamp) +def is_scanner_lib_built(libname): + """Return True if ScannerBit/lib/lib.so exists, i.e. the GAMBIT + runtime interface library for this scanner has been built. Independent + of any external dependency (libdiver.so etc.) that may also be required + — this is just the GAMBIT-side library that ScannerBit dlopens.""" + return os.path.exists(os.path.join( + _REPO, "ScannerBit", "lib", "lib" + libname + ".so")) + + def parse_excluded_yaml(path): """Return {libname: [reason, ...]} for every excluded library, where libname is the bare library identifier (e.g. "scanner_great", "scanner_python").""" @@ -175,11 +184,15 @@ def make_tag(label, color): return color + label + RESET + " " * (TAG_W - len(label)) # Build (name, status_kind, info-string) tuples for the grouped - # (native+external) section. status_kind is one of: + # (native+external) section. The Status column answers "is this + # scanner ready to run right now?" — so [installed] requires every + # prerequisite to be in place: for an external scanner that means + # both the ExternalProject build (so libdiver.so etc. exists) and + # ScannerBit's own runtime interface library (libscanner_.so); + # for a native scanner it just means the latter. # "disabled" — every version is excluded - # "installed" — at least one external version's stamp exists - # "not_installed" — has external make targets but none built yet - # "" — native-only / nothing actionable to display + # "installed" — at least one version is fully ready to run + # "not_installed" — none ready, but user actions can fix it grouped_rows = [] for name in sorted(grouped, key=str.lower): versions = sorted(grouped[name], key=lambda v: v["version"]) @@ -187,13 +200,22 @@ def make_tag(label, color): external_versions = [v for v in versions if v["target"]] native_versions = [v for v in versions if not v["target"]] + def is_version_ready(v): + if v["disabled"]: + return False + if not is_scanner_lib_built(v["library"]): + return False + if v["target"] and not is_target_installed(build_dir, v["target"]): + return False + return True + parts = [] if external_versions: tgt_strs = [] for v in external_versions: if v["disabled"] and not all_disabled: tgt_strs.append("{} [disabled]".format(v["target"])) - elif is_target_installed(build_dir, v["target"]): + elif is_version_ready(v): tgt_strs.append("{} [installed]".format(v["target"])) else: tgt_strs.append(v["target"]) @@ -202,24 +224,20 @@ def make_tag(label, color): parts.append("targets: " + DIM + "none; native GAMBIT scanner" + RESET) info = " ".join(parts) - any_installed = any( - (not v["disabled"]) and is_target_installed(build_dir, v["target"]) - for v in external_versions - ) + any_ready = any(is_version_ready(v) for v in versions) if all_disabled: kind = "disabled" - elif any_installed: + elif any_ready: kind = "installed" - elif external_versions: - kind = "not_installed" else: - kind = "" + kind = "not_installed" grouped_rows.append((name, kind, info)) # Python rows: one per scanner. Status is libscanner_python lib status (if # excluded, all are disabled) overlaid with per-scanner Python module # availability. py_lib_disabled = "scanner_python" in excluded + py_lib_built = is_scanner_lib_built("scanner_python") python_rows = [] for line in python_scanner_lines: parts = line.split("|") @@ -229,8 +247,17 @@ def make_tag(label, color): pstatus = parts[1] # Restore "+" → ", " in the missing-pkgs field (see python_scanners.cmake). pmissing = parts[2].replace("+", ", ") if len(parts) > 2 else "" - disabled = py_lib_disabled or (pstatus != "enabled") - python_rows.append((pname, disabled, pmissing)) + # "Ready to run" requires libscanner_python.so to exist and the + # required pip packages to be installed; "disabled" means the + # configure-time decision was that this scanner can't be built or + # used at all (libscanner_python excluded, or pip pkgs missing). + if py_lib_disabled or pstatus != "enabled": + kind = "disabled" + elif py_lib_built: + kind = "installed" + else: + kind = "not_installed" + python_rows.append((pname, kind, pmissing)) # Compute column widths across both sections so they line up. all_names = [r[0] for r in grouped_rows] + [r[0] for r in python_rows] @@ -269,10 +296,9 @@ def make_tag(label, color): n=name, nw=name_w, tag=tag, info=info)) # Print Python rows (sorted alphabetically for parity with the grouped section). - for name, disabled, missing in sorted(python_rows, key=lambda r: r[0].lower()): - kind = "disabled" if disabled else "" + for name, kind, missing in sorted(python_rows, key=lambda r: r[0].lower()): tag = make_tag(tag_label[kind], tag_color[kind]) - if disabled: + if kind == "disabled": if missing: body = "none; python plugin – install package(s) [{}] to enable".format(missing) else: From fe72d951f2872742d2c6f3c265388fcdffd5da09 Mon Sep 17 00:00:00 2001 From: Anders Kvellestad Date: Sat, 9 May 2026 16:00:15 +0200 Subject: [PATCH 09/12] Conflict between -DBits and Ditch now produces error message at cmake time. --- cmake/utilities.cmake | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/cmake/utilities.cmake b/cmake/utilities.cmake index 14eb6c94ae..40814c549a 100644 --- a/cmake/utilities.cmake +++ b/cmake/utilities.cmake @@ -754,6 +754,30 @@ function(gambit_translate_bits_into_itch) return() endif() + # A Bit can't be both kept (-DBits) and ditched (-Ditch). Error rather + # than silently letting -Ditch win, which would otherwise quietly drop a + # Bit the user explicitly asked to keep. + set(_conflicts "") + foreach(_b ${Bits}) + string(TOLOWER "${_b}" _b_lower) + foreach(_i ${itch}) + string(TOLOWER "${_i}" _i_lower) + if(_b_lower STREQUAL _i_lower) + list(APPEND _conflicts "${_b}") + endif() + endforeach() + endforeach() + if(_conflicts) + list(REMOVE_DUPLICATES _conflicts) + string(REPLACE ";" ", " _conflicts_csv "${_conflicts}") + message(FATAL_ERROR + "\nThe following entries appear in both -DBits and -Ditch: " + "${_conflicts_csv}.\n" + "A Bit cannot be simultaneously kept and ditched. Please remove " + "the conflicting entries from one of the two arguments and " + "reconfigure.\n") + endif() + message("${BoldYellow}-- Bits=\"${Bits}\" requested; restricting build to these Bits (+ ScannerBit).${ColourReset}") # Warn about -DBits entries that don't match a real Bit directory. From 7783bb04e853d3f3d37f5ead1f9e1f662e134d9e Mon Sep 17 00:00:00 2001 From: Anders Kvellestad Date: Sat, 9 May 2026 17:13:20 +0200 Subject: [PATCH 10/12] Reduce overly verbose code comments and simplified some code --- Backends/scripts/list_backends.py | 95 ++++++-------------- CMakeLists.txt | 21 ++--- ScannerBit/scripts/list_scanners.py | 131 ++++++++-------------------- Utils/scripts/harvesting_tools.py | 91 +++++++------------ Utils/scripts/list_table.py | 40 +++++++++ cmake/utilities.cmake | 42 +++++---- 6 files changed, 155 insertions(+), 265 deletions(-) create mode 100644 Utils/scripts/list_table.py diff --git a/Backends/scripts/list_backends.py b/Backends/scripts/list_backends.py index 09b54a80d8..f5d8f34bca 100644 --- a/Backends/scripts/list_backends.py +++ b/Backends/scripts/list_backends.py @@ -1,15 +1,7 @@ #!/usr/bin/env python3 -"""Print a status table for every known GAMBIT backend interface. -Reads config/gambit_backend_interfaces.yaml (regenerated by -backend_harvester.py) for the enabled/disabled interface status, and the data file written by -CMakeLists.txt at configure time (TARGETS=) for the set of -active backend make targets. Walks every Bit's include/src/examples tree -to find which Bits reference each backend. Prints one row per canonical -BACKENDNAME with its status, a "used by:" list of Bits, and the -backend's user-invokable make targets. -""" -from __future__ import print_function - +"""Print a status table (one row per canonical BACKENDNAME) for every known +GAMBIT backend interface. Reads config/gambit_backend_interfaces.yaml plus +a data file written by CMakeLists.txt at configure time.""" import io import os import sys @@ -17,12 +9,14 @@ import yaml _HERE = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, os.path.normpath(os.path.join(_HERE, "..", "..", "Utils", "scripts"))) +_REPO = os.path.normpath(os.path.join(_HERE, "..", "..")) +sys.path.insert(0, os.path.join(_REPO, "Utils", "scripts")) from harvesting_tools import ( # noqa: E402 compute_bits_per_backend, get_all_backendnames, ) +import list_table # noqa: E402 def main(): @@ -43,15 +37,11 @@ def main(): ditched_set = {s for s in line[len("DITCHED="):].rstrip("\n").split(";") if s} def is_ditched(canonical): - """Return True if any cmake ${itch} entry refers to this canonical - backend. Matches case-insensitively, accepts both bare names - (e.g. 'feynhiggs') and versioned names (e.g. 'feynhiggs_2.11.2'), - and tolerates the underscore-collapsed form (e.g. itch entry - 'susyhit' matching canonical 'SUSY_HIT'). The interfaces YAML may - still mark this backend "enabled" (BOSSed types must remain in the - rollcall even when the backend itself is excluded), but if cmake - has ditched it then there is no make target to build it and it - cannot actually be used in this configuration.""" + # Match canonical (or its underscore-collapsed form, for cases like + # SUSY_HIT vs susyhit_1.5) against any cmake ${itch} entry, with or + # without a trailing _. Catches BOSSed backends that the + # interfaces YAML keeps "enabled" for type availability after the + # build itself was excluded. cl = canonical.lower() cl_collapsed = canonical.replace('_', '').lower() for entry in ditched_set: @@ -63,24 +53,21 @@ def is_ditched(canonical): return False def is_target_installed(target): - """ExternalProject_Add writes a -done stamp into its stamp dir - once configure/build/install all succeed; gambit's clean/nuke targets - remove it, so the stamp tracks reality.""" if not build_dir or not target: return False return os.path.exists(os.path.join( build_dir, target + "-prefix", "src", target + "-stamp", target + "-done")) - with io.open("./config/gambit_backend_interfaces.yaml", "r") as f: + with io.open(os.path.join(_REPO, "config", "gambit_backend_interfaces.yaml"), "r") as f: bdata = yaml.safe_load(f) or {} enabled_set = set((bdata.get("enabled") or {}).keys()) - frontend_dir = "./Backends/include/gambit/Backends/frontends" + frontend_dir = os.path.join(_REPO, "Backends", "include", "gambit", "Backends", "frontends") all_backends = get_all_backendnames(frontend_dir) - bits_per = compute_bits_per_backend(".", frontend_dir) + bits_per = compute_bits_per_backend(_REPO, frontend_dir) - # Group each active target under its longest-matching canonical (so e.g. + # Group each target under its longest-matching canonical (so e.g. # darksusy_MSSM_6.4.0 lands under DarkSUSY_MSSM, not under DarkSUSY). canonicals_by_len = sorted(all_backends, key=len, reverse=True) targets_by_canonical = {b: [] for b in all_backends} @@ -92,9 +79,8 @@ def is_target_installed(target): targets_by_canonical[canonical].append(t) matched = True break - # Fallback: try the collapsed-underscore form, for canonicals where - # the make target drops internal underscores (e.g. SUSY_HIT - # -> susyhit_1.5). + # Fallback for canonicals whose make target drops internal underscores + # (SUSY_HIT -> susyhit_1.5). if not matched: for canonical in canonicals_by_len: cl = canonical.replace("_", "").lower() @@ -105,12 +91,6 @@ def is_target_installed(target): rows = [] for canonical in sorted(all_backends, key=lambda s: s.lower()): targets = sorted(targets_by_canonical[canonical]) - # A backend is "active" iff its frontend interface compiles in (per - # the YAML's enabled list) AND cmake hasn't ditched the build path - # in the current configuration. The latter check catches cases like - # FeynHiggs (gfortran>=10) and BOSSed backends like Pythia/Rivet - # that the interfaces YAML keeps as "enabled" for type-availability - # reasons even after they've been auto-excluded. active = (canonical in enabled_set) and not is_ditched(canonical) bits = bits_per.get(canonical, []) rows.append((canonical, active, bits, targets)) @@ -122,46 +102,25 @@ def is_target_installed(target): # bits string. Bump to fit the "Used by Bits" header if narrower. used_by_col_w = max(9 + bits_w, len("Used by Bits")) bits_w = used_by_col_w - 9 - use_color = sys.stdout.isatty() - YELLOW = "\033[33m" if use_color else "" - GREEN = "\033[32m" if use_color else "" - CYAN = "\033[36m" if use_color else "" - DIM = "\033[2m" if use_color else "" - BOLD = "\033[1m" if use_color else "" - RESET = "\033[0m" if use_color else "" - - # Status column tags: width 15 to fit "[not installed]" (longest visible - # label). ANSI codes don't add visible width, so explicit space-padding - # after the colored bracket keeps every row aligned. - TAG_W = len("[not installed]") - def make_tag(label, color): - if not label: - return " " * TAG_W - return color + label + RESET + " " * (TAG_W - len(label)) - - # Header row + dashed separator. print(" {bold}{h1:<{nw}} {h2:<{tw}} {h3:<{w3}} {h4}{reset}".format( - bold=BOLD, reset=RESET, + bold=list_table.BOLD, reset=list_table.RESET, h1="Backend", nw=name_w, - h2="Status", tw=TAG_W, + h2="Status", tw=list_table.TAG_W, h3="Used by Bits", w3=used_by_col_w, h4="Make targets")) print(" {0} {1} {2} {3}".format( - "-" * name_w, "-" * TAG_W, "-" * used_by_col_w, "-" * len("Make targets"))) + "-" * name_w, "-" * list_table.TAG_W, + "-" * used_by_col_w, "-" * len("Make targets"))) for name, active, bits, targets in rows: - # Bits column: dim "none" when empty. ANSI codes don't take visible - # width, so do the column-padding manually rather than via {:<{bw}}. if bits: bits_visible = ", ".join(bits) bits_field = bits_visible + " " * (bits_w - len(bits_visible)) else: - bits_field = DIM + "none" + RESET + " " * (bits_w - len("none")) + bits_field = list_table.DIM + "none" + list_table.RESET + " " * (bits_w - len("none")) - # Targets column: dim "none" when no targets; otherwise annotate - # individually-installed targets inline. if not targets: - targets_field = DIM + "none" + RESET + targets_field = list_table.DIM + "none" + list_table.RESET else: ann = [] for t in targets: @@ -179,13 +138,9 @@ def make_tag(label, color): kind = "not_installed" else: kind = "" - tag_color = {"disabled": YELLOW, "installed": GREEN, - "not_installed": CYAN, "": ""}[kind] - tag_label = {"disabled": "[disabled]", "installed": "[installed]", - "not_installed": "[not installed]", "": ""}[kind] - tag = make_tag(tag_label, tag_color) print(" {name:<{nw}} {tag} used by: {bits} targets: {tgts}".format( - name=name, nw=name_w, tag=tag, bits=bits_field, tgts=targets_field)) + name=name, nw=name_w, tag=list_table.tag_for_kind(kind), + bits=bits_field, tgts=targets_field)) if __name__ == "__main__": diff --git a/CMakeLists.txt b/CMakeLists.txt index 9e39f244c2..2eecf20a56 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -632,16 +632,11 @@ list(APPEND MODULE_HARVESTER_FILES "${PROJECT_SOURCE_DIR}/config/resolution_type set(MODULE_HARVESTER_FILES ${MODULE_HARVESTER_FILES} ${BACKEND_HARVESTER_FILES}) remove_build_files(models_harvested backends_harvested modules_harvested printers_harvested colliders_harvested) -# Add backends and scanners. (Done before the harvest custom-target -# registration and update_cmakelists invocation below so that any late -# additions to ${itch} from cmake/backends.cmake / cmake/scanners.cmake — -# e.g. the FeynHiggs ditch for gfortran>=10 — are reflected in the -# itch_with_commas string we pass to those steps. Otherwise the backend -# harvester would still mark such backends "enabled" in -# gambit_backend_interfaces.yaml even though no make targets exist to build them.) +# Add backends and scanners. Done before the harvest registration so late +# additions to ${itch} (e.g. the FeynHiggs gfortran>=10 ditch) reach the +# harvester. include(cmake/externals.cmake) -# Reprocess the ditch set into a comma-separated list (final, post-externals). string (REPLACE ";" "," itch_with_commas "${itch}") if(EXISTS "${PROJECT_SOURCE_DIR}/Elements/") @@ -715,9 +710,8 @@ endif() # Print the list of backend make targets that survived the ditch / opt-in pass. gambit_print_available_backend_targets() -# Register a `make list-backends` target that prints a status table of every -# known backend, including which Bits use it. The data file is rewritten on -# each cmake configure to reflect the current set of active targets. +# `make list-backends` and `make list-scanners` status-table targets. +# Each writes its data file on every cmake configure. file(WRITE "${PROJECT_BINARY_DIR}/list_backends_data.txt" "TARGETS=${GAMBIT_AVAILABLE_BACKEND_TARGETS}\nBUILD_DIR=${PROJECT_BINARY_DIR}\nDITCHED=${itch}\n") add_custom_target(list-backends @@ -726,11 +720,6 @@ add_custom_target(list-backends USES_TERMINAL) add_dependencies(list-backends backend_harvest) -# Register a `make list-scanners` target that prints an analogous status table -# for scanner plugins (external, native, and Python). The data file is -# rewritten on each cmake configure; the script also reads -# scratch/build_time/scanbit_excluded_libs.yaml (written by ScannerBit) and -# walks ScannerBit/src/scanners/ for the canonical native plugin list. file(WRITE "${PROJECT_BINARY_DIR}/list_scanners_data.txt" "EXTERNAL_TARGETS=${GAMBIT_AVAILABLE_SCANNER_TARGETS}\nPYTHON_SCANNERS=${GAMBIT_PYTHON_SCANNER_STATUS}\nBUILD_DIR=${PROJECT_BINARY_DIR}\n") add_custom_target(list-scanners diff --git a/ScannerBit/scripts/list_scanners.py b/ScannerBit/scripts/list_scanners.py index bedc05b232..b94e999f9c 100644 --- a/ScannerBit/scripts/list_scanners.py +++ b/ScannerBit/scripts/list_scanners.py @@ -1,20 +1,8 @@ #!/usr/bin/env python3 -"""Print a status table of every known GAMBIT scanner plugin. - -Combines three data sources: - * the data file written by CMakeLists.txt at configure time (active external - scanner make targets and per-Python-scanner status), passed in argv[1]; - * scratch/build_time/scanbit_excluded_libs.yaml (disabled native libs), - produced by ScannerBit/CMakeLists.txt; - * the scanner_plugin(name, version(...)) registrations under - ScannerBit/src/scanners/, which are the canonical list of native plugins. - -Prints one row per canonical plugin name. External plugins list their -versioned make targets; native-only plugins are tagged 'native'; Python -scanners are tagged 'python'. -""" -from __future__ import print_function - +"""Print a status table of every known GAMBIT scanner plugin (one row per +canonical plugin name). Reads scratch/build_time/scanbit_excluded_libs.yaml +plus a data file written by CMakeLists.txt at configure time, and walks +ScannerBit/src/scanners/ for native scanner_plugin() registrations.""" import io import os import re @@ -25,6 +13,9 @@ _HERE = os.path.dirname(os.path.abspath(__file__)) _REPO = os.path.normpath(os.path.join(_HERE, "..", "..")) +sys.path.insert(0, os.path.join(_REPO, "Utils", "scripts")) + +import list_table # noqa: E402 def parse_data_file(path): @@ -43,10 +34,6 @@ def parse_data_file(path): def is_target_installed(build_dir, target): - """An ExternalProject_Add target writes a -done stamp into its - stamp dir once configure/build/install all succeed. The stamp persists - across cmake reconfigures and is removed by the corresponding clean/nuke - targets, so it tracks reality.""" if not build_dir or not target: return False stamp = os.path.join(build_dir, target + "-prefix", "src", @@ -55,24 +42,19 @@ def is_target_installed(build_dir, target): def is_scanner_lib_built(libname): - """Return True if ScannerBit/lib/lib.so exists, i.e. the GAMBIT - runtime interface library for this scanner has been built. Independent - of any external dependency (libdiver.so etc.) that may also be required - — this is just the GAMBIT-side library that ScannerBit dlopens.""" return os.path.exists(os.path.join( _REPO, "ScannerBit", "lib", "lib" + libname + ".so")) def parse_excluded_yaml(path): - """Return {libname: [reason, ...]} for every excluded library, where libname - is the bare library identifier (e.g. "scanner_great", "scanner_python").""" + """Return {libname: [reason, ...]} for every excluded scanner/objective + library in scanbit_excluded_libs.yaml.""" excluded = {} if not os.path.exists(path): return excluded with io.open(path, "r") as f: data = yaml.safe_load(f) or {} for libfile, info in data.items(): - # libfile looks like "libscanner_great.so" or "libobjective_python.so". name = libfile if name.startswith("lib"): name = name[3:] @@ -81,9 +63,9 @@ def parse_excluded_yaml(path): raw_reasons = info.get("reason") or [] if isinstance(raw_reasons, str): raw_reasons = [raw_reasons] - # Each reason was emitted by ScannerBit/CMakeLists.txt as a literal - # YAML sequence entry like '- file missing: "ROOT"', which the YAML - # parser turns into a single-key mapping. Flatten back to "key: value". + # ScannerBit emits each reason as a literal YAML entry like + # '- file missing: "ROOT"', which yaml.safe_load parses as a + # single-key mapping; flatten back to "key: value". reasons = [] for r in raw_reasons: if isinstance(r, dict): @@ -103,12 +85,9 @@ def parse_excluded_yaml(path): def parse_native_plugins(scanners_dir): - """Walk ScannerBit/src/scanners//*.cpp and collect every - scanner_plugin(name, version(...)) registration. Returns a list of dicts: - { 'libname': 'scanner_', 'plugin': '', 'version': '' }. - - Skips the python/ directory; Python scanners come from a separate data - source (cmake/python_scanners.cmake).""" + """Walk ScannerBit/src/scanners//* and return a list of + {libname, plugin, version} dicts derived from scanner_plugin(...) + registrations. Skips python/ — those come from a separate data source.""" rows = [] if not os.path.isdir(scanners_dir): return rows @@ -144,18 +123,16 @@ def main(): ) native = parse_native_plugins(os.path.join(_REPO, "ScannerBit", "src", "scanners")) - # Map external make target to its companion plugin library, e.g. - # "diver_1.3.0" -> "scanner_diver_1.3.0". Most ScannerBit/src/scanners/ - # subdirs include the version suffix, but a few (e.g. "great") do not, so - # also register a versionless fallback mapping ("scanner_great"). + # Map "" -> "scanner_" plus a versionless fallback for + # canonical dirs that drop the version (e.g. ScannerBit/src/scanners/great). + # Versionless slot is first-seen wins, which is fine because in practice + # only one scanner family hits this fallback. target_for_lib = {} for tgt in external_targets: target_for_lib.setdefault("scanner_" + tgt, tgt) if "_" in tgt: target_for_lib.setdefault("scanner_" + tgt.rsplit("_", 1)[0], tgt) - # Group native+external plugins by canonical plugin name. Each entry is a - # list of per-version dicts, sorted by version. grouped = {} for r in native: libname = r["libname"] @@ -168,34 +145,16 @@ def main(): "reason": "; ".join(excluded.get(libname, [])), }) - use_color = sys.stdout.isatty() - YELLOW = "\033[33m" if use_color else "" - GREEN = "\033[32m" if use_color else "" - CYAN = "\033[36m" if use_color else "" - DIM = "\033[2m" if use_color else "" - RESET = "\033[0m" if use_color else "" - - # Status column tags: width 15 to fit the longest visible label - # ("[not installed]"). ANSI color codes don't take visible width. - TAG_W = len("[not installed]") - def make_tag(label, color): - if not label: - return " " * TAG_W - return color + label + RESET + " " * (TAG_W - len(label)) - - # Build (name, status_kind, info-string) tuples for the grouped - # (native+external) section. The Status column answers "is this - # scanner ready to run right now?" — so [installed] requires every - # prerequisite to be in place: for an external scanner that means - # both the ExternalProject build (so libdiver.so etc. exists) and - # ScannerBit's own runtime interface library (libscanner_.so); - # for a native scanner it just means the latter. - # "disabled" — every version is excluded - # "installed" — at least one version is fully ready to run - # "not_installed" — none ready, but user actions can fix it + # [installed] = ready to run: external scanners need both the + # ExternalProject build and libscanner_.so; native scanners need + # only the latter. + def _ver_key(v): + return [int(c) if c.isdigit() else c + for c in re.split('([0-9]+)', v["version"])] + grouped_rows = [] for name in sorted(grouped, key=str.lower): - versions = sorted(grouped[name], key=lambda v: v["version"]) + versions = sorted(grouped[name], key=_ver_key) all_disabled = all(v["disabled"] for v in versions) external_versions = [v for v in versions if v["target"]] native_versions = [v for v in versions if not v["target"]] @@ -221,7 +180,7 @@ def is_version_ready(v): tgt_strs.append(v["target"]) parts.append("targets: " + ", ".join(tgt_strs)) if native_versions and not external_versions: - parts.append("targets: " + DIM + "none; native GAMBIT scanner" + RESET) + parts.append("targets: " + list_table.DIM + "none; native GAMBIT scanner" + list_table.RESET) info = " ".join(parts) any_ready = any(is_version_ready(v) for v in versions) @@ -247,10 +206,6 @@ def is_version_ready(v): pstatus = parts[1] # Restore "+" → ", " in the missing-pkgs field (see python_scanners.cmake). pmissing = parts[2].replace("+", ", ") if len(parts) > 2 else "" - # "Ready to run" requires libscanner_python.so to exist and the - # required pip packages to be installed; "disabled" means the - # configure-time decision was that this scanner can't be built or - # used at all (libscanner_python excluded, or pip pkgs missing). if py_lib_disabled or pstatus != "enabled": kind = "disabled" elif py_lib_built: @@ -259,45 +214,27 @@ def is_version_ready(v): kind = "not_installed" python_rows.append((pname, kind, pmissing)) - # Compute column widths across both sections so they line up. all_names = [r[0] for r in grouped_rows] + [r[0] for r in python_rows] if not all_names: print(" (no scanner plugins found)") return name_w = max(max(len(n) for n in all_names), len("Scanner")) - # Header row + dashed separator. - BOLD = "\033[1m" if use_color else "" print(" {bold}{h1:<{nw}} {h2:<{tw}} {h3}{reset}".format( - bold=BOLD, reset=RESET, + bold=list_table.BOLD, reset=list_table.RESET, h1="Scanner", nw=name_w, - h2="Status", tw=TAG_W, + h2="Status", tw=list_table.TAG_W, h3="Make targets")) print(" {0} {1} {2}".format( - "-" * name_w, "-" * TAG_W, "-" * len("Make targets"))) - - tag_color = { - "disabled": YELLOW, - "installed": GREEN, - "not_installed": CYAN, - "": "", - } - tag_label = { - "disabled": "[disabled]", - "installed": "[installed]", - "not_installed": "[not installed]", - "": "", - } + "-" * name_w, "-" * list_table.TAG_W, "-" * len("Make targets"))) # Print grouped (native+external) rows. for name, kind, info in grouped_rows: - tag = make_tag(tag_label[kind], tag_color[kind]) print(" {n:<{nw}} {tag} {info}".format( - n=name, nw=name_w, tag=tag, info=info)) + n=name, nw=name_w, tag=list_table.tag_for_kind(kind), info=info)) # Print Python rows (sorted alphabetically for parity with the grouped section). for name, kind, missing in sorted(python_rows, key=lambda r: r[0].lower()): - tag = make_tag(tag_label[kind], tag_color[kind]) if kind == "disabled": if missing: body = "none; python plugin – install package(s) [{}] to enable".format(missing) @@ -305,9 +242,9 @@ def is_version_ready(v): body = "none; python plugin – install package(s) to enable" else: body = "none; python plugin" - info = "targets: " + DIM + body + RESET + info = "targets: " + list_table.DIM + body + list_table.RESET print(" {n:<{nw}} {tag} {info}".format( - n=name, nw=name_w, tag=tag, info=info)) + n=name, nw=name_w, tag=list_table.tag_for_kind(kind), info=info)) if __name__ == "__main__": diff --git a/Utils/scripts/harvesting_tools.py b/Utils/scripts/harvesting_tools.py index 5ac4ff5bcb..9ccf5e7262 100644 --- a/Utils/scripts/harvesting_tools.py +++ b/Utils/scripts/harvesting_tools.py @@ -717,15 +717,9 @@ def get_all_backendnames(frontend_dir): def derive_used_backends(enabled_bits, project_root, frontend_dir): - """Return the set of backends used by any of the enabled Bits, derived - via rollcall-macro analysis (BACKEND_REQ / LONG_BACKEND_REQ capabilities - matched against frontend BE_FUNCTION/BE_VARIABLE/BE_CONV_FUNCTION - declarations, plus NEEDS_CLASSES_FROM for BOSSed-class deps, plus the - hard-coded overrides in _KNOWN_BIT_USES_OVERRIDES). - - This mirrors the resolution that the GAMBIT dependency resolver - performs at runtime, so it doesn't keep backends alive merely because - their name shows up in a doc comment or a string literal.""" + """Return the backends used by any enabled Bit, derived from the same + rollcall-macro analysis that compute_bits_per_backend uses (no + token-sweeping over comments / string literals).""" if not enabled_bits: return set() bits_per_backend = compute_bits_per_backend(project_root, frontend_dir) @@ -744,45 +738,30 @@ def derive_optin_backend_excludes(enabled_bits, project_root, return auto_excluded, used, all_backends -# Hard-coded overrides for backends whose Bit attribution can't be derived -# statically from rollcall macros. Two cases today: -# * DDCalc: DarkBit calls into DDCalc via LONG_BACKEND_REQ with capability -# names assembled by C-preprocessor concatenation (CAT_3(EXPERIMENT,_,NAME)). -# A static parser can't resolve those token-pastes. -# * HepLikeData: pure-data backend; FlavBit depends on it structurally (data -# files at runtime) but never references it through BACKEND_REQ / -# NEEDS_CLASSES_FROM. -# If a future Bit starts/stops using one of these, update this map. +# Bit attribution that can't be derived statically from rollcall macros: +# DDCalc — DarkBit references its capabilities via LONG_BACKEND_REQ +# with names assembled by C-preprocessor token pasting. +# HepLikeData — pure-data backend, no rollcall-macro reference at all. _KNOWN_BIT_USES_OVERRIDES = { "DDCalc": ["DarkBit"], "HepLikeData": ["FlavBit"], } -_CXX_BLOCK_COMMENT_RE = re.compile(r'/\*.*?\*/', re.DOTALL) -_CXX_LINE_COMMENT_RE = re.compile(r'//[^\n]*') - -def _strip_cxx_comments(text): - """Approximate C++ comment removal good enough for token-level scans of - GAMBIT source. Not perfect inside string literals, but backends are not - referenced via string literals here, so the trade-off is safe.""" - return _CXX_LINE_COMMENT_RE.sub('', _CXX_BLOCK_COMMENT_RE.sub('', text)) - _FRONTEND_SHARED_INCLUDE_RE = re.compile( r'^\s*#\s*include\s+"(gambit/Backends/frontends/shared_includes/[^"]+)"', re.MULTILINE) def _read_frontend_recursive(path, backends_inc_dir, _seen=None): - """Read a frontend .hpp, strip comments, and inline any - 'gambit/Backends/frontends/shared_includes/...' headers it pulls in - (e.g. MicrOmegas singlet-DM variants share BE_* declarations through a - common file). Returns the concatenated text.""" + """Read a frontend .hpp with comments stripped, inlining any + shared_includes/ headers it pulls in (e.g. MicrOmegas singlet variants + share BE_* declarations through a common file).""" if _seen is None: _seen = set() if path in _seen or not os.path.exists(path): return "" _seen.add(path) try: with io.open(path, "r", encoding="utf-8", errors="replace") as f: - text = _strip_cxx_comments(f.read()) + text = comment_remover(f.read()) except (IOError, OSError): return "" out = text @@ -794,11 +773,8 @@ def _read_frontend_recursive(path, backends_inc_dir, _seen=None): _BE_MACRO_RE = re.compile(r'\bBE_(?:FUNCTION|VARIABLE|CONV_FUNCTION)\s*\(') def _extract_be_capabilities(text): - """Return the set of capability names a frontend file declares. The - capability is the last quoted-string argument inside BE_FUNCTION, - BE_VARIABLE, or BE_CONV_FUNCTION; the macro args may contain nested - parentheses (e.g. C++ argument-type lists), so we walk to the matching - closing paren rather than using a flat regex.""" + # Capability is the last quoted-string arg of each BE_* macro; nested + # parens (e.g. C++ argument-type lists) preclude a flat regex. caps = set() for m in _BE_MACRO_RE.finditer(text): depth, i = 1, m.end() @@ -816,30 +792,25 @@ def _extract_be_capabilities(text): def compute_bits_per_backend(project_root, frontend_dir): - """Return a dict mapping each canonical BACKENDNAME to the sorted list of - Bit names that depend on it, using GAMBIT's rollcall macros as the - source of truth. - - A Bit is attributed to a backend if either: - * the Bit's rollcall headers contain (LONG_)BACKEND_REQ for any - capability the backend's frontend declares via BE_FUNCTION, - BE_VARIABLE, or BE_CONV_FUNCTION (following shared_includes), or - * the Bit's headers contain NEEDS_CLASSES_FROM(, ...), - used for BOSSed-class dependencies. - - A small hard-coded override map (_KNOWN_BIT_USES_OVERRIDES) covers the - cases where capability names are constructed via C-preprocessor token - pasting at the call site, or where the dependency is purely structural - (data files only, no rollcall-macro reference).""" + """Return {canonical BACKENDNAME -> sorted list of Bit names that depend + on it}, derived from rollcall macros: a Bit is attributed to a backend + if the Bit's rollcall headers contain (LONG_)BACKEND_REQ for any + capability the backend's frontend declares via BE_FUNCTION/VARIABLE/ + CONV_FUNCTION (following shared_includes), or NEEDS_CLASSES_FROM(). + _KNOWN_BIT_USES_OVERRIDES patches in cases the macros can't express.""" all_backends = get_all_backendnames(frontend_dir) if not all_backends: return {} - # Step 1: each backend's frontend capability set, with shared-include inlining. - # frontend_dir = "/Backends/include/gambit/Backends/frontends" - # 3 levels up gets us to "/Backends/include" — the include root - # used by the shared_includes/ path. - backends_inc_dir = os.path.normpath(os.path.join(frontend_dir, "..", "..", "..")) + # Frontend headers can pull in shared includes via paths rooted at + # Backends/include — derive that root rather than assuming a layout. + backends_inc_dir = frontend_dir + while backends_inc_dir and os.path.basename(backends_inc_dir) != "include": + parent = os.path.dirname(backends_inc_dir) + if parent == backends_inc_dir: + break + backends_inc_dir = parent + caps_by_backend = {b: set() for b in all_backends} if os.path.isdir(frontend_dir): for fn in sorted(os.listdir(frontend_dir)): @@ -852,7 +823,8 @@ def compute_bits_per_backend(project_root, frontend_dir): os.path.join(frontend_dir, fn), backends_inc_dir) caps_by_backend[backend] |= _extract_be_capabilities(text) - # Step 2: each Bit's requested capabilities and NEEDS_CLASSES_FROM names. + # Substring match on 'Bit' is loose but harmless: only Bit dirs ever + # carry rollcall headers with the macros we look for. bit_dirs = sorted(d for d in os.listdir(project_root) if 'Bit' in d and os.path.isdir(os.path.join(project_root, d))) result = {b: set() for b in all_backends} @@ -868,7 +840,7 @@ def compute_bits_per_backend(project_root, frontend_dir): try: with io.open(os.path.join(root, fn), "r", encoding="utf-8", errors="replace") as f: - text = _strip_cxx_comments(f.read()) + text = comment_remover(f.read()) except (IOError, OSError): continue for m in _BIT_REQ_RE.finditer(text): bit_caps.add(m.group(1)) @@ -877,7 +849,6 @@ def compute_bits_per_backend(project_root, frontend_dir): if (caps_by_backend[be] & bit_caps) or (be in bit_ncfs): result[be].add(bit) - # Step 3: apply hard-coded overrides. for be, bits in _KNOWN_BIT_USES_OVERRIDES.items(): if be in result: result[be].update(bits) diff --git a/Utils/scripts/list_table.py b/Utils/scripts/list_table.py new file mode 100644 index 0000000000..e0a2d29015 --- /dev/null +++ b/Utils/scripts/list_table.py @@ -0,0 +1,40 @@ +"""Shared formatting helpers for `make list-backends` / `make list-scanners`. +ANSI codes carry no visible width, so callers that pad table columns around +make_tag()/tag_for_kind() must compute padding from the visible labels.""" +import sys + + +_use_color = sys.stdout.isatty() + +YELLOW = "\033[33m" if _use_color else "" +GREEN = "\033[32m" if _use_color else "" +CYAN = "\033[36m" if _use_color else "" +DIM = "\033[2m" if _use_color else "" +BOLD = "\033[1m" if _use_color else "" +RESET = "\033[0m" if _use_color else "" + +# Status column width: fits the longest label, "[not installed]". +TAG_W = len("[not installed]") + +_TAG_LABEL = { + "disabled": "[disabled]", + "installed": "[installed]", + "not_installed": "[not installed]", + "": "", +} +_TAG_COLOR = { + "disabled": YELLOW, + "installed": GREEN, + "not_installed": CYAN, + "": "", +} + + +def make_tag(label, color): + if not label: + return " " * TAG_W + return color + label + RESET + " " * (TAG_W - len(label)) + + +def tag_for_kind(kind): + return make_tag(_TAG_LABEL[kind], _TAG_COLOR[kind]) diff --git a/cmake/utilities.cmake b/cmake/utilities.cmake index 40814c549a..bf5bf4e81b 100644 --- a/cmake/utilities.cmake +++ b/cmake/utilities.cmake @@ -749,6 +749,20 @@ endmacro() # Call before retrieve_bits; reads the cmake variable `Bits` and the # already-populated ${ALL_GAMBIT_BITS}. ScannerBit is implicitly kept; # any explicit -Ditch entries also stand. +# Sets ${out_var} to YES if ${token} matches any of ${ARGN} (case-insensitive). +function(_gambit_list_contains_ci out_var token) + string(TOLOWER "${token}" _t_lower) + set(${out_var} NO PARENT_SCOPE) + foreach(_x ${ARGN}) + string(TOLOWER "${_x}" _x_lower) + if(_x_lower STREQUAL _t_lower) + set(${out_var} YES PARENT_SCOPE) + return() + endif() + endforeach() +endfunction() + + function(gambit_translate_bits_into_itch) if(NOT DEFINED Bits OR "${Bits}" STREQUAL "") return() @@ -759,13 +773,10 @@ function(gambit_translate_bits_into_itch) # Bit the user explicitly asked to keep. set(_conflicts "") foreach(_b ${Bits}) - string(TOLOWER "${_b}" _b_lower) - foreach(_i ${itch}) - string(TOLOWER "${_i}" _i_lower) - if(_b_lower STREQUAL _i_lower) - list(APPEND _conflicts "${_b}") - endif() - endforeach() + _gambit_list_contains_ci(_in_itch "${_b}" ${itch}) + if(_in_itch) + list(APPEND _conflicts "${_b}") + endif() endforeach() if(_conflicts) list(REMOVE_DUPLICATES _conflicts) @@ -783,14 +794,7 @@ function(gambit_translate_bits_into_itch) # Warn about -DBits entries that don't match a real Bit directory. set(_unknown_Bits "") foreach(_b ${Bits}) - string(TOLOWER "${_b}" _b_lower) - set(_match_found NO) - foreach(_d ${ALL_GAMBIT_BITS}) - string(TOLOWER "${_d}" _d_lower) - if(_d_lower STREQUAL _b_lower) - set(_match_found YES) - endif() - endforeach() + _gambit_list_contains_ci(_match_found "${_b}" ${ALL_GAMBIT_BITS}) if(NOT _match_found) list(APPEND _unknown_Bits "${_b}") endif() @@ -805,13 +809,7 @@ function(gambit_translate_bits_into_itch) foreach(_d ${ALL_GAMBIT_BITS}) string(TOLOWER "${_d}" _d_lower) if(NOT _d_lower STREQUAL "scannerbit") - set(_keep NO) - foreach(_b ${Bits}) - string(TOLOWER "${_b}" _b_lower) - if(_d_lower STREQUAL _b_lower) - set(_keep YES) - endif() - endforeach() + _gambit_list_contains_ci(_keep "${_d}" ${Bits}) if(NOT _keep) list(APPEND _added_to_itch ${_d}) set(_local_itch "${_local_itch};${_d}") From f284f85009dcc04d1163bd3bfa4bbbcb03093be5 Mon Sep 17 00:00:00 2001 From: Anders Kvellestad Date: Sun, 10 May 2026 17:43:02 +0200 Subject: [PATCH 11/12] Removed an unnecessary backend list at the end of the cmake output. --- BUILD_OPTIONS.md | 27 ++++++++++++++------------- CMakeLists.txt | 6 +++--- cmake/externals.cmake | 2 +- cmake/utilities.cmake | 12 ------------ 4 files changed, 18 insertions(+), 29 deletions(-) diff --git a/BUILD_OPTIONS.md b/BUILD_OPTIONS.md index cb0d47d2ec..1840613b9c 100644 --- a/BUILD_OPTIONS.md +++ b/BUILD_OPTIONS.md @@ -18,24 +18,14 @@ For a more complete list of cmake variables, take a look in the file `CMakeCache -Ditch="ColliderBit;NeutrinoBit;Mathematica" # Select only the Bits you need: Bits -# ScannerBit is always kept implicitly. Combines with -Ditch: -# anything explicitly ditched stays ditched even if listed in -DBits. +# Similar to -Ditch, but opt-in rather than opt-out. If some Bits +# are specified using -DBits, all other Bits except ScannerBit are +# added to the ditch list. -DBits and -Ditch must not overlap. -DBits="DarkBit;PrecisionBit;SpecBit;DecayBit" # typical dark matter project -DBits="ColliderBit;PrecisionBit;SpecBit;DecayBit" # typical collider project -DBits="CosmoBit;DarkBit" # typical cosmology project -# Skip the build of any backend interface that no enabled Bit references -# at the source level: GAMBIT_TRIM_BACKEND_INTERFACES (On|Off) -# Auto-set to ON when -DBits is used or when -Ditch removes any Bit; forced ON -# below to opt in even when keeping every Bit. Set to OFF to disable the -# trimming explicitly (e.g. when developing a new backend that no Bit yet -# references). Pair with -DGAMBIT_FORCE_BACKEND_INTERFACE="Name1;Name2" to -# keep specific backends regardless. --DGAMBIT_TRIM_BACKEND_INTERFACES=On --DGAMBIT_FORCE_BACKEND_INTERFACE="Acropolis;DarkCast" - - # List the FlexibleSUSY models to build: BUILD_FS_MODELS # The names of the available FlexibleSUSY models correspond to # the subdirectories in @@ -106,5 +96,16 @@ For a more complete list of cmake variables, take a look in the file `CMakeCache # Create Graphviz files: HAVE_GRAPHVIZ (On|Off) -DHAVE_GRAPHVIZ=On + +# Skip the build of any backend interface that no enabled Bit references +# at the source level: GAMBIT_TRIM_BACKEND_INTERFACES (On|Off) +# Auto-set to ON when -DBits is used or when -Ditch removes any Bit. +# Set this to OFF to disable the trimming explicitly, e.g. when developing +# a new backend that no Bit yet references. Pair with +# -DGAMBIT_FORCE_BACKEND_INTERFACE="Name1;Name2" to keep specific backends +# regardless. +-DGAMBIT_TRIM_BACKEND_INTERFACES=On +-DGAMBIT_FORCE_BACKEND_INTERFACE="Acropolis;DarkCast" + ``` diff --git a/CMakeLists.txt b/CMakeLists.txt index 2eecf20a56..a551bb56e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -707,9 +707,6 @@ if(EXISTS "${PROJECT_SOURCE_DIR}/ScannerBit/") include(${PROJECT_BINARY_DIR}/linkedout.cmake) endif() -# Print the list of backend make targets that survived the ditch / opt-in pass. -gambit_print_available_backend_targets() - # `make list-backends` and `make list-scanners` status-table targets. # Each writes its data file on every cmake configure. file(WRITE "${PROJECT_BINARY_DIR}/list_backends_data.txt" @@ -726,3 +723,6 @@ add_custom_target(list-scanners COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/ScannerBit/scripts/list_scanners.py ${PROJECT_BINARY_DIR}/list_scanners_data.txt WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} USES_TERMINAL) + +message("${BoldYellow}-- Run `make list-backends` to see the available backend make targets.${ColourReset}") +message("${BoldYellow}-- Run `make list-scanners` to see the available scanner make targets.${ColourReset}") diff --git a/cmake/externals.cmake b/cmake/externals.cmake index c3e4c88e55..a6602e022e 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -158,7 +158,7 @@ macro(add_extra_targets type package ver dir dl target) endif() - # Track available backend make targets for gambit_print_available_backend_targets. + # Track available backend make targets for the list-backends helper. if (${type} STREQUAL "backend") list(APPEND GAMBIT_AVAILABLE_BACKEND_TARGETS "${package}_${ver}") elseif (${type} STREQUAL "backend model") diff --git a/cmake/utilities.cmake b/cmake/utilities.cmake index bf5bf4e81b..93c9c79b9e 100644 --- a/cmake/utilities.cmake +++ b/cmake/utilities.cmake @@ -955,15 +955,3 @@ function(gambit_configure_optin_build) set(BACKEND_HARVESTER_EXTRA_ARGS "${_extra_args}" PARENT_SCOPE) set(update_cmakelists_extra_args "${_extra_args}" PARENT_SCOPE) endfunction() - - -# Function to print the comma-separated list of backend make targets that -# survived the ditch / opt-in pass. The list is populated as a side-effect -# of add_extra_targets in cmake/externals.cmake, so call this after -# cmake/externals.cmake has been included. -function(gambit_print_available_backend_targets) - set(_targets "${GAMBIT_AVAILABLE_BACKEND_TARGETS}") - list(SORT _targets) - string(REPLACE ";" ", " _targets_csv "${_targets}") - message("${BoldYellow}-- Available backend make targets: ${_targets_csv}${ColourReset}") -endfunction() \ No newline at end of file From 2c9008fe9e82e58c50ad7d9d46993a12a5281051 Mon Sep 17 00:00:00 2001 From: Anders Kvellestad Date: Sun, 10 May 2026 18:13:02 +0200 Subject: [PATCH 12/12] One more tweak to status listing of external scanners. --- ScannerBit/scripts/list_scanners.py | 36 ++++++++++++++++------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/ScannerBit/scripts/list_scanners.py b/ScannerBit/scripts/list_scanners.py index b94e999f9c..4114881f2e 100644 --- a/ScannerBit/scripts/list_scanners.py +++ b/ScannerBit/scripts/list_scanners.py @@ -145,9 +145,9 @@ def main(): "reason": "; ".join(excluded.get(libname, [])), }) - # [installed] = ready to run: external scanners need both the - # ExternalProject build and libscanner_.so; native scanners need - # only the latter. + # [installed] = the user's build action for this scanner kind has + # completed: external scanners → ExternalProject stamp present; + # native scanners → libscanner_.so built. def _ver_key(v): return [int(c) if c.isdigit() else c for c in re.split('([0-9]+)', v["version"])] @@ -162,11 +162,17 @@ def _ver_key(v): def is_version_ready(v): if v["disabled"]: return False - if not is_scanner_lib_built(v["library"]): - return False - if v["target"] and not is_target_installed(build_dir, v["target"]): - return False - return True + if v["target"]: + return is_target_installed(build_dir, v["target"]) + return is_scanner_lib_built(v["library"]) + + any_ready = any(is_version_ready(v) for v in versions) + if all_disabled: + kind = "disabled" + elif any_ready: + kind = "installed" + else: + kind = "not_installed" parts = [] if external_versions: @@ -180,16 +186,12 @@ def is_version_ready(v): tgt_strs.append(v["target"]) parts.append("targets: " + ", ".join(tgt_strs)) if native_versions and not external_versions: - parts.append("targets: " + list_table.DIM + "none; native GAMBIT scanner" + list_table.RESET) + body = "none; native GAMBIT scanner" + if kind == "not_installed": + body += " – build GAMBIT to install" + parts.append("targets: " + list_table.DIM + body + list_table.RESET) info = " ".join(parts) - any_ready = any(is_version_ready(v) for v in versions) - if all_disabled: - kind = "disabled" - elif any_ready: - kind = "installed" - else: - kind = "not_installed" grouped_rows.append((name, kind, info)) # Python rows: one per scanner. Status is libscanner_python lib status (if @@ -240,6 +242,8 @@ def is_version_ready(v): body = "none; python plugin – install package(s) [{}] to enable".format(missing) else: body = "none; python plugin – install package(s) to enable" + elif kind == "not_installed": + body = "none; python plugin – all packages available, build GAMBIT to install" else: body = "none; python plugin" info = "targets: " + list_table.DIM + body + list_table.RESET