Verify pre-print licences instead of assuming CC-BY; make every shipped skill findable

.github/workflows/validate.yml

@@ -50,6 +50,14 @@ jobs:
       - name: Run test suite
         run: python -m pytest tests/ -q
 
+      # -- Gate 0b: Index coverage -------------------------------------------
+      # A skill absent from skills_index.json / kb_bundle.json ships on disk but
+      # cannot be found by search, the MCP server, or the docs site. Nothing else
+      # notices: check_license_tiers.py only inspects skills the index already
+      # knows about, so the orphan stays invisible and CI stays green.
+      - name: Every skill is in every index its collection publishes
+        run: python -m scripts.skill_index
+
       # -- Gate 10: Plugin manifest validates --------------------------------
       - name: Validate .claude-plugin/marketplace.json
         run: |

.gitignore

@@ -43,3 +43,7 @@ outputs/
 # License clarification wave output (runtime artifact — not committed)
 governance/license_clarification/wave.yaml
 .license-cache-v2.json
+
+# Local resolver caches (preprint licence lookups, embeddings). Never committed.
+.cache/
+**/.cache/

collections/metabolomics/v2/kb_bundle.json

@@ -57256,6 +57256,17 @@
       ],
       "license_tier": "open"
     },
+    "masster": {
+      "dois": [],
+      "kb_slugs": [],
+      "license_tier": "noncommercial",
+      "repo_urls": [
+        "https://github.com/zamboni-lab/masster-dist"
+      ],
+      "tools": [
+        "Masster"
+      ]
+    },
     "match-factor-threshold-filtering": {
       "dois": [
         "10.1371/journal.pone.0306202"

collections/metabolomics/v2/skills/masster/SKILL.md

@@ -8,6 +8,7 @@ metadata:
   techniques:
   - LC-MS
   repo_url: https://github.com/zamboni-lab/masster-dist
+  license_tier: noncommercial
   tool_license:
     tier: noncommercial
     requires_ack: true

collections/metabolomics/v2/skills_index.json

@@ -63355,6 +63355,21 @@
   ],
   "license_tier": "open"
  },
+ {
+  "slug": "masster",
+  "name": "masster",
+  "description": "Use when you need to run the Zamboni-lab Masster (MASSter) workflow for untargeted LC-MS metabolomics data analysis. NONCOMMERCIAL tool — confirm permitted use before applying (see License notice).",
+  "edam_operation": null,
+  "edam_topics": [],
+  "tools": [
+   "Masster"
+  ],
+  "dois": [],
+  "techniques": [
+   "LC-MS"
+  ],
+  "license_tier": "noncommercial"
+ },
  {
   "slug": "match-factor-threshold-filtering",
   "name": "match-factor-threshold-filtering",

docs-site/build_search_index.py

@@ -33,6 +33,9 @@
 
 # Repo root = parent of docs-site/
 REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT))
+
+from scripts.skill_index import split_frontmatter  # noqa: E402  (needs REPO_ROOT on the path)
 DOCS_SITE = Path(__file__).resolve().parent
 OUTPUT = DOCS_SITE / "search_index.json"
 
@@ -62,24 +65,16 @@ def _load_yaml(path: Path) -> dict | None:
 
 
 def _parse_skill_md(path: Path) -> tuple[dict | None, str]:
-    """Return (frontmatter_dict, body_text). Body is everything after the
-    closing '---' delimiter; frontmatter is parsed as YAML."""
+    """Return (frontmatter_dict, body_text) via the canonical SKILL.md parser."""
     try:
         raw = path.read_text(encoding="utf-8")
     except OSError as exc:
         sys.stderr.write(f"warn: failed to read {path}: {exc}\n")
         return None, ""
-    if not raw.startswith("---"):
-        return None, raw
-    parts = raw.split("---", 2)
-    if len(parts) < 3:
-        return None, raw
-    try:
-        fm = yaml.safe_load(parts[1])
-    except yaml.YAMLError as exc:
-        sys.stderr.write(f"warn: bad frontmatter in {path}: {exc}\n")
-        fm = None
-    return fm, parts[2]
+    frontmatter, body = split_frontmatter(raw)
+    if frontmatter is None:
+        sys.stderr.write(f"warn: bad frontmatter in {path}\n")
+    return frontmatter, body
 
 
 def _extract_summary(body: str) -> str:

governance/LICENSE_TIERS.md

@@ -24,3 +24,28 @@ bundles (referenced, never embedded). Beyond that, they differ:
 - **`restricted`** instead carries a **non-blocking soft note**: "no clear license
   detected — verify before commercial use or redistribution." Absence of a license
   is an unknown, not an explicit prohibition, so no blocking gate is required.
+
+## The source-reuse axis
+
+`license_tier` answers *"what may I do with the tool?"*. It does **not** answer
+*"what may we do with the source text?"* — and the two disagree. CC-BY-ND permits
+commercial use of a tool (so it is not `noncommercial`) while forbidding derivative
+text (so it is not `open`). Reading a paper's licence through the tool tier would
+call such a source open, which is the mistake the blanket "pre-prints are always
+CC-BY" rule made.
+
+The second canonical table, `source_reuse` in `license_tiers.yaml`, answers the
+source question. Reach it via `scripts/license_tier.py::source_reuse_for_license`.
+
+| Value | Meaning | Examples |
+|---|---|---|
+| `full` | Redistribute, quote at length, derive — the open-access bar | CC-BY, CC-BY-SA, CC0 |
+| `limited` | Some grant, but not full reuse; link-only | CC-BY-NC-*, CC-BY-ND-* |
+| `none` | No reuse rights granted | arXiv `nonexclusive-distrib` |
+| *(absent)* | **Unknown** — blocks admission, reported loudly | any unlisted licence |
+
+An unlisted licence returns `None`, never a default. `None` (unknown) and `none`
+(a known refusal) are different answers and must not be collapsed: one means we
+failed to establish the rights, the other means the rights were withheld. Only
+`full` admits a source at an open `access.type`. See
+`scripts/preprint_license.py` and `governance/OPEN_ACCESS_POLICY.md` § Pre-prints.

governance/OPEN_ACCESS_POLICY.md

@@ -148,15 +148,28 @@ The static page at <https://holobiomicslab.github.io/asb-skill-collections/paper
 
 Pre-prints (bioRxiv, medRxiv, ChemRxiv, arXiv) may be included, but **their posting licence must be verified per pre-print — it is not always CC-BY.** bioRxiv and medRxiv let the author choose CC-BY, CC-BY-NC, CC-BY-ND, CC-BY-NC-ND, CC0 or "no reuse allowed"; arXiv's default licence-to-distribute grants no reuse rights at all. Treating every pre-print as CC-BY would admit text this project has no right to redistribute.
 
-Resolve the actual licence before admitting an entry, and record how it was resolved:
+Resolve the actual licence before admitting an entry, with `scripts/preprint_license.py`:
 
-| Server | Where the licence lives | Record as |
-|---|---|---|
-| bioRxiv / medRxiv | `https://api.biorxiv.org/details/<server>/<doi>` → `license` (`cc_by`, `cc_by_nc_nd`, `cc_no`, …) | `verified_via: biorxiv_api_license` |
-| arXiv | Atom API entry → `<license>` | `verified_via: arxiv_api_license` |
-| ChemRxiv | item metadata → `license` | `verified_via: chemrxiv_api_license` |
+```bash
+python -m scripts.preprint_license --doi 10.1101/2020.04.09.033894
+python -m scripts.preprint_license --corpus 'collections/*/v*/corpus.yaml'
+```
+
+**Read pre-print-ness from the DOI registry, never from a DOI prefix.** A prefix identifies a *registrant*, not a work type — `10.1101` is Cold Spring Harbor, covering bioRxiv, medRxiv **and** CSHL journals, and bioRxiv/medRxiv now also register under the **openRxiv prefix `10.64898`**. Both registries declare the work type directly, so any server they cover resolves with no code change:
+
+| Registry | Declares a pre-print by | Licence field | Record as |
+|---|---|---|---|
+| Crossref (bioRxiv, medRxiv, ChemRxiv, most servers) | `type: posted-content` | `license[].URL` | `verified_via: crossref_license` |
+| DataCite (arXiv) | `types.resourceTypeGeneral: Preprint` | `rightsList[].rightsUri` | `verified_via: datacite_license` |
+
+Two corrections to earlier guidance, both established by testing the live APIs:
+
+- **arXiv's Atom API returns no licence at all.** Use DataCite (or OAI-PMH `arXivRaw`). arXiv's default `nonexclusive-distrib` licence grants **no reuse rights**, so an arXiv pre-print is not open by default.
+- **ChemRxiv's public API is behind Cloudflare and answers `403` to scripted requests.** Resolve ChemRxiv through Crossref, which carries its licence.
+
+`api.biorxiv.org/details/<server>/<doi>` remains an authoritative cross-check for bioRxiv/medRxiv (returning tokens such as `cc_by`, `cc_by_nc_nd`, `cc_no`), but it is server-specific and so is not the primary path.
 
-Only a licence permitting the intended reuse admits the pre-print at an open `access.type`. A non-reuse licence makes the entry link-only, exactly as for a closed paper. Note that bioRxiv/medRxiv DOIs now carry the **openRxiv prefix `10.64898`** alongside the legacy Cold Spring Harbor prefix `10.1101`; both are genuine.
+Only a **full-reuse** licence — one mapped to `source_reuse: full` in [`license_tiers.yaml`](license_tiers.yaml), i.e. CC-BY, CC-BY-SA or CC0 — admits a pre-print at an open `access.type`. NC and ND licences grant `limited` reuse and make the entry link-only, exactly as for a closed paper. A licence the table does not know resolves to **unknown**, which blocks admission and must be reported; it is never silently treated as either permissive or restrictive.
 
 Per [`CONTENT_POLICY.md`](CONTENT_POLICY.md) §3, `preprint` is a **provenance** value and must never be used as an `access.type`. When the peer-reviewed version is later published with a different DOI, the corpus entry is updated to point at the canonical DOI (linking back to the pre-print as `source_history`).
 

governance/license_tiers.yaml

@@ -42,7 +42,51 @@ spdx:
   CC-BY-NC-4.0: noncommercial
   CC-BY-NC-3.0: noncommercial
   CC-BY-NC-SA-4.0: noncommercial
+  CC-BY-NC-ND-4.0: noncommercial
   PolyForm-Noncommercial-1.0.0: noncommercial
+  # NoDerivatives permits commercial *use* of the tool but forbids derivative
+  # distribution. It carries no explicit commercial prohibition, so it is not
+  # `noncommercial`; it grants no derivative rights, so it is not `open`.
+  CC-BY-ND-4.0: restricted
 fallback:
   noncommercial_keywords: [noncommercial, "non-commercial", "cc-by-nc", polyform-noncommercial]
   default: restricted
+
+# ---------------------------------------------------------------------------
+# SOURCE-REUSE axis. Distinct from the `spdx` tier map above.
+#
+#   spdx:         "what may a consumer do with the TOOL?"      -> license_tier
+#   source_reuse: "what may WE do with the SOURCE TEXT?"       -> access.type
+#
+# These are different questions and must not be conflated (LICENSE_TIERS.md).
+# CC-BY-ND allows commercial use of a tool but forbids derivative text, so it is
+# `restricted` on the tool axis and `limited` on the source axis.
+#
+#   full    - redistribute, quote at length, and derive (the OA-first bar)
+#   limited - some grant, but not full reuse (NC and/or ND); link-only
+#   none    - no reuse rights granted at all
+#
+# An SPDX id absent from this table resolves to UNKNOWN, never to a default.
+# Unknown must block admission and be reported loudly; see scripts/preprint_license.py.
+source_reuse:
+  CC-BY-1.0: full
+  CC-BY-2.0: full
+  CC-BY-2.5: full
+  CC-BY-3.0: full
+  CC-BY-4.0: full
+  CC-BY-SA-3.0: full
+  CC-BY-SA-4.0: full
+  CC0-1.0: full
+  CC-BY-NC-3.0: limited
+  CC-BY-NC-4.0: limited
+  CC-BY-NC-SA-3.0: limited
+  CC-BY-NC-SA-4.0: limited
+  CC-BY-ND-3.0: limited
+  CC-BY-ND-4.0: limited
+  CC-BY-NC-ND-3.0: limited
+  CC-BY-NC-ND-4.0: limited
+  # arXiv's default: a non-exclusive licence to distribute, granting no reuse rights.
+  arXiv-1.0: none
+  # A pre-print server displays the work but the author reserved all rights
+  # (bioRxiv/medRxiv token `cc_no`). A known refusal, not an unknown.
+  NoReuse-1.0: none

governance/preprint_servers.yaml

@@ -0,0 +1,29 @@
+# Server-specific licence APIs, consulted only when the DOI registry itself
+# declares no usable licence URL. Crossref names the server in `institution[].name`;
+# that name (lowercased) selects an entry here. Nothing dispatches on a DOI prefix,
+# so adding a server is a change to this file, never to scripts/preprint_license.py.
+#
+# Human doc: governance/OPEN_ACCESS_POLICY.md, section "Pre-prints".
+version: 1
+
+servers:
+  biorxiv:
+    detail_url: https://api.biorxiv.org/details/biorxiv/{doi}
+  medrxiv:
+    detail_url: https://api.biorxiv.org/details/medrxiv/{doi}
+
+# The token these servers return in `collection[].license` -> an SPDX id.
+# Resolve the id's reuse rights through `source_reuse` in license_tiers.yaml.
+# A token absent from this table is UNKNOWN and blocks admission; it is never
+# assumed to be permissive.
+license_tokens:
+  cc_by: CC-BY-4.0
+  cc_by_sa: CC-BY-SA-4.0
+  cc_by_nc: CC-BY-NC-4.0
+  cc_by_nd: CC-BY-ND-4.0
+  cc_by_nc_sa: CC-BY-NC-SA-4.0
+  cc_by_nc_nd: CC-BY-NC-ND-4.0
+  cc0: CC0-1.0
+  # The author reserved all rights: bioRxiv displays the preprint but grants no
+  # reuse. Distinct from an unknown licence -- this is a known refusal.
+  cc_no: NoReuse-1.0

scripts/check_license_tiers.py

@@ -9,9 +9,11 @@
 
 import yaml
 
-from scripts.license_tier import ack_required
+from scripts.license_tier import ack_required, load_map
 
-_VALID = {"open", "noncommercial", "restricted"}
+# The tier vocabulary has one home: governance/license_tiers.yaml. A hand-copied
+# set here silently rejects any tier added there.
+_VALID = set(load_map()["tiers"])
 
 
 def check_collection(collection_dir) -> list[str]:

scripts/license_tier.py

@@ -51,3 +51,29 @@ def ack_required(tier: str) -> bool:
     """Only the noncommercial tier triggers a blocking runtime use acknowledgment.
     'restricted' is labeled + link-only with a soft note, but does not block."""
     return tier == "noncommercial"
+
+
+def source_reuse_for_license(spdx: str, _map: dict | None = None) -> str | None:
+    """What may we do with a SOURCE's text under this licence?
+
+    Returns 'full', 'limited', 'none', or None when the licence is not in the
+    canonical table. None means UNKNOWN and must block admission -- it is never
+    the same as 'none', which is a known refusal. See governance/license_tiers.yaml.
+    """
+    if not spdx or not spdx.strip():
+        return None
+    table = (_map or load_map()).get("source_reuse") or {}
+    key = spdx.strip().lower()
+    for known, reuse in table.items():
+        if known.lower() == key:
+            return reuse
+    return None
+
+
+def permits_full_reuse(spdx: str, _map: dict | None = None) -> bool:
+    """True only for licences that grant redistribution and derivation.
+
+    An unknown licence is not full reuse, but callers must still distinguish the
+    two: use source_reuse_for_license() when the difference matters.
+    """
+    return source_reuse_for_license(spdx, _map) == "full"