From bb8d5e19f4ddbe7e38b69cc40ea4de2d81ad8609 Mon Sep 17 00:00:00 2001 From: MTRNord Date: Fri, 3 Jul 2026 22:58:56 +0200 Subject: [PATCH] add CI to refresh client and server ecosystem data and validate the setup on new client and server ecosystem PRs Signed-off-by: MTRNord --- .github/scripts/check_ecosystem_pr.py | 161 +++++ .github/scripts/refresh_ecosystem_data.py | 627 +++++++++++++++++++ .github/workflows/ecosystem-entry-check.yml | 45 ++ .github/workflows/refresh-ecosystem-data.yml | 105 ++++ .gitignore | 3 + CONTENT.md | 26 + doc/ecosystem-doap-extension.md | 138 ++++ 7 files changed, 1105 insertions(+) create mode 100644 .github/scripts/check_ecosystem_pr.py create mode 100644 .github/scripts/refresh_ecosystem_data.py create mode 100644 .github/workflows/ecosystem-entry-check.yml create mode 100644 .github/workflows/refresh-ecosystem-data.yml create mode 100644 doc/ecosystem-doap-extension.md diff --git a/.github/scripts/check_ecosystem_pr.py b/.github/scripts/check_ecosystem_pr.py new file mode 100644 index 0000000000..d5ea5ba41e --- /dev/null +++ b/.github/scripts/check_ecosystem_pr.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""PR-time check for new/changed ecosystem entries (see pr-bot.yml counterpart +ecosystem-entry-check.yml). Reads changed files from the PR head via `gh api` +(never checks out or executes code from the fork) and reuses +refresh_ecosystem_data's own TOML/DOAP parsing to flag entries with no +repo/doap_url, and to validate any doap_url that was set or changed. Posts (or +updates) a single PR comment via `gh pr comment --edit-last --create-if-none`. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tomllib +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) +from refresh_ecosystem_data import ( # noqa: E402 + CLIENT_CONFIG, + SERVER_CONFIG, + CategoryConfig, + TomlTable, + repo_host, + split_frontmatter, + split_toml_array, + validate_doap_url, +) + +REPO = os.environ["GITHUB_REPOSITORY"] +PR_NUMBER = os.environ["PR_NUMBER"] +HEAD_SHA = os.environ["HEAD_SHA"] +DOC_LINK = "doc/ecosystem-doap-extension.md" + + +def gh(*args: str) -> str: + result = subprocess.run(["gh", *args, "-R", REPO], capture_output=True, text=True) + result.check_returncode() + return result.stdout + + +def changed_files() -> list[str]: + return gh("pr", "diff", PR_NUMBER, "--name-only").splitlines() + + +def fetch_at_head(path: str) -> str | None: + result = subprocess.run( + ["gh", "api", f"repos/{REPO}/contents/{path}", "-f", f"ref={HEAD_SHA}"], + capture_output=True, + text=True, + ) + if result.returncode != 0: + return None # file was deleted in this PR + content: str = json.loads(result.stdout)["content"] + return content + + +def decode_base64(content: str) -> str: + import base64 + + return base64.b64decode(content).decode("utf-8") + + +def check_entry(location: str, fields: TomlTable, config: CategoryConfig) -> list[str]: + lines = [] + repo_url = fields.get(config.repo_field) + doap_url = fields.get(config.doap_field) + + if not repo_url and not doap_url: + lines.append( + f"**{location}**: no `{config.repo_field}` or `{config.doap_field}` set, " + f"so this entry won't be picked up by the automated refresh. Add a " + f"GitHub/GitLab repo link, or publish a DOAP file and set `{config.doap_field}` " + f"for full sync (see [{DOC_LINK}]({DOC_LINK}))." + ) + return lines + + if repo_url and repo_host(repo_url) is None: + lines.append( + f"**{location}**: `{config.repo_field}` isn't on GitHub or GitLab, so it " + f"won't get the automated archived-repo check (dead-link checking still " + f"applies to any host) — that's fine, it's only skipped, not an error." + ) + + if doap_url: + issues = validate_doap_url(doap_url) + if issues: + formatted = "\n".join(f" - {issue}" for issue in issues) + lines.append( + f"**{location}**: `{config.doap_field}` has issues:\n{formatted}" + ) + + return lines + + +def check_clients(paths: list[str]) -> list[str]: + lines = [] + for path in paths: + if not path.startswith("content/ecosystem/clients/") or path.endswith( + "_index.md" + ): + continue + content = fetch_at_head(path) + if content is None: + continue + split = split_frontmatter(decode_base64(content)) + if split is None: + continue + _, body, _ = split + fields = tomllib.loads(body).get("extra", {}) + lines += check_entry(Path(path).name, fields, CLIENT_CONFIG) + return lines + + +def check_servers(paths: list[str]) -> list[str]: + servers_path = "content/ecosystem/servers/servers.toml" + if servers_path not in paths: + return [] + content = fetch_at_head(servers_path) + if content is None: + return [] + _, blocks = split_toml_array(decode_base64(content), "servers") + lines = [] + for block in blocks: + entry = tomllib.loads(block)["servers"][0] + lines += check_entry(entry.get("name", "unknown"), entry, SERVER_CONFIG) + return lines + + +def main() -> int: + paths = changed_files() + lines = check_clients(paths) + check_servers(paths) + + if not lines: + return 0 # nothing to say; leave any earlier comment as-is + + body = ( + "### Automated ecosystem entry check\n\n" + + "\n\n".join(lines) + + "\n\n" + ) + subprocess.run( + [ + "gh", + "pr", + "comment", + PR_NUMBER, + "-R", + REPO, + "--edit-last", + "--create-if-none", + "--body", + body, + ], + check=True, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/refresh_ecosystem_data.py b/.github/scripts/refresh_ecosystem_data.py new file mode 100644 index 0000000000..3d6fc19460 --- /dev/null +++ b/.github/scripts/refresh_ecosystem_data.py @@ -0,0 +1,627 @@ +#!/usr/bin/env python3 +"""Refresh ecosystem client/server data: repo archived status, dead links, and +diffs from a maintainer's optional `doap_url` file (doc/ecosystem-doap-extension.md). +Only flat fields (maturity, licence, room) are rewritten; everything else is reported +for a human to apply. Run weekly by refresh-ecosystem-data.yml. +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import tomllib +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any +from urllib.parse import quote, unquote, urlparse +from xml.etree import ElementTree as ET + +TomlTable = dict[str, Any] + +CLIENTS_DIR = Path("content/ecosystem/clients") +SERVERS_FILE = Path("content/ecosystem/servers/servers.toml") + +GITHUB_API = "https://api.github.com/repos/{owner}/{repo}" +GITLAB_API = "https://gitlab.com/api/v4/projects/{path}" +USER_AGENT = "matrix-org-ecosystem-refresh/1.0" + +DOAP_NS = { + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "doap": "http://usefulinc.com/ns/doap#", + "foaf": "http://xmlns.com/foaf/0.1/", + "matrix": "https://matrix.org/ns/ecosystem#", + # Client vs. server feature support live in their own namespaces rather than + # a shared element with a type attribute, so which one applies is inherent + # to the tag rather than something to check separately. + "matrix_client": "https://matrix.org/ns/ecosystem/client#", + "matrix_server": "https://matrix.org/ns/ecosystem/server#", +} +# "unknown" is deliberately not a value here - it's what a feature means when a +# DOAP file simply doesn't mention it, not something to assert explicitly. +FEATURE_STATUSES = {"supported", "partial", "unsupported"} +FEATURE_KINDS = ("client", "server") +FEATURE_PREFIX = {"client": "matrix_client", "server": "matrix_server"} +# https://spec.matrix.org/v1.18/appendices/#matrix-uri-scheme +MATRIX_URI_ROOM_SIGILS = {"r": "#", "roomid": "!"} +# Keep in sync with the feature keys under [extra.features] documented in CONTENT.md. +KNOWN_FEATURES: dict[str, set[str]] = { + "client": { + "e2ee", + "spaces", + "voip_1to1", + "threads", + "sso", + "voip_jitsi", + "multi_account", + "multi_language", + "oauth", + "invisible_crypto", + "voip_matrixrtc", + "sliding_sync", + }, + "server": set(), # servers don't have a published feature matrix yet +} + + +@dataclass +class CategoryConfig: + kind: str # "client" or "server" - selects which matrix:SupportedFeature apply + repo_field: str + licence_field: str + maturity_field: str + doap_field: str + room_field: str + + +CLIENT_CONFIG = CategoryConfig( + kind="client", + repo_field="repo", + licence_field="licence", + maturity_field="maturity", + doap_field="doap_url", + room_field="matrix_room", +) +SERVER_CONFIG = CategoryConfig( + kind="server", + repo_field="repository", + licence_field="licence", + maturity_field="maturity", + doap_field="doap_url", + room_field="room", +) + + +@dataclass +class FieldEdit: + field: str + old: str | None + new: str + reason: str + + +@dataclass +class EntryReport: + location: str + edits: list[FieldEdit] = field(default_factory=list) + findings: list[str] = field(default_factory=list) + + +# HTTP + + +def http_get( + url: str, headers: dict[str, str] | None = None, timeout: float = 10.0 +) -> tuple[int, bytes]: + """GET a URL. Returns (0, b"") on network errors rather than raising.""" + request = urllib.request.Request( + url, headers={"User-Agent": USER_AGENT, **(headers or {})} + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.status, response.read() + except urllib.error.HTTPError as error: + return error.code, b"" + except (urllib.error.URLError, TimeoutError): + return 0, b"" + + +def is_dead_link(url: str) -> bool | None: + """True if the URL is confirmed gone (404/410). None if we can't tell.""" + status, _ = http_get(url) + if status == 0: + return None + return status in (404, 410) + + +# Repo archived-status check - GitHub/GitLab only. Other forges and VCS systems +# (Codeberg, sourcehut, self-hosted Gitea, ...) are skipped here, not treated as +# an error; dead-link checking above still covers them regardless of host. + + +def repo_host(url: str) -> tuple[str, str] | None: + parsed = urlparse(url) + path = parsed.path.strip("/") + if not path: + return None + if parsed.netloc == "github.com": + parts = path.split("/") + if len(parts) >= 2: + return "github", f"{parts[0]}/{parts[1]}" + elif parsed.netloc == "gitlab.com": + return "gitlab", path + return None + + +def is_archived(repo_url: str, github_token: str | None) -> bool | None: + host = repo_host(repo_url) + if host is None: + return None + kind, ident = host + if kind == "github": + owner, name = ident.split("/", 1) + headers = {"Accept": "application/vnd.github+json"} + if github_token: + headers["Authorization"] = f"Bearer {github_token}" + status, body = http_get(GITHUB_API.format(owner=owner, repo=name), headers) + else: + status, body = http_get(GITLAB_API.format(path=quote(ident, safe=""))) + if status != 200: + return None + return bool(json.loads(body).get("archived", False)) + + +# DOAP + + +def slugify(value: str) -> str: + return re.sub(r"[^A-Za-z0-9_.-]+", "_", value).strip("_") or "entry" + + +def fetch_doap_bytes(url: str, cache_key: str, cache_dir: Path) -> bytes | None: + """Fetch a DOAP file, using the cache only to bridge transient failures.""" + cache_file = cache_dir / f"{slugify(cache_key)}.xml" + status, body = http_get(url) + if status == 200 and body: + cache_file.write_bytes(body) + return body + if status in (404, 410): + # Authoritative "gone" (e.g. removed for moderation) - never resurrect + # deliberately removed content from the cache. + cache_file.unlink(missing_ok=True) + return None + if cache_file.exists(): + return cache_file.read_bytes() + return None + + +def parse_matrix_room_uri(uri: str) -> str | None: + """ "matrix:r/foo:example.org" -> "#foo:example.org" (or matrix:roomid/... -> "!..."). + Returns None for anything that isn't a room-alias/room-id matrix: URI.""" + parsed = urlparse(uri) + if parsed.scheme != "matrix": + return None + segments = parsed.path.strip("/").split("/") + if len(segments) != 2: + return None + kind, identifier = segments + sigil = MATRIX_URI_ROOM_SIGILS.get(kind) + if not sigil or not identifier: + return None + return sigil + unquote(identifier) + + +def validate_doap(root: ET.Element) -> list[str]: + errors = [] + project = root.find("doap:Project", DOAP_NS) + if project is None: + return ["missing element"] + for kind in FEATURE_KINDS: + prefix = FEATURE_PREFIX[kind] + for supported in project.findall(f"{prefix}:SupportedFeature", DOAP_NS): + feature = supported.get("feature") + status = supported.get("status") + if not feature: + errors.append( + f"{prefix}:SupportedFeature is missing the feature attribute" + ) + if status not in FEATURE_STATUSES: + errors.append( + f"{prefix}:SupportedFeature has invalid status {status!r}" + ) + return errors + + +def doap_warnings(root: ET.Element) -> list[str]: + """Non-fatal issues: flagged for review but don't invalidate the whole file.""" + warnings: list[str] = [] + project = root.find("doap:Project", DOAP_NS) + if project is None: + return warnings + + room = project.find("matrix:room", DOAP_NS) + if ( + room is not None + and room.text + and parse_matrix_room_uri(room.text.strip()) is None + ): + warnings.append( + f"matrix:room {room.text.strip()!r} isn't a valid matrix: URI (expected e.g. " + f"'matrix:r/your-room:example.org' - see " + f"https://spec.matrix.org/v1.18/appendices/#matrix-uri-scheme) - ignored" + ) + + for kind in FEATURE_KINDS: + prefix = FEATURE_PREFIX[kind] + for supported in project.findall(f"{prefix}:SupportedFeature", DOAP_NS): + name = supported.get("feature") + if name and name not in KNOWN_FEATURES[kind]: + warnings.append( + f"{prefix}:SupportedFeature declares unknown feature {name!r} " + f"(check for typos - it will be ignored)" + ) + return warnings + + +def parse_doap(root: ET.Element) -> TomlTable | None: + project = root.find("doap:Project", DOAP_NS) + if project is None: + return None + + def text(tag: str) -> str | None: + element = project.find(tag, DOAP_NS) + return element.text.strip() if element is not None and element.text else None + + def resource(tag: str) -> str | None: + element = project.find(tag, DOAP_NS) + return ( + element.get(f"{{{DOAP_NS['rdf']}}}resource") + if element is not None + else None + ) + + licence = None + licence_resource = resource("doap:license") + spdx_prefix = "https://spdx.org/licenses/" + if licence_resource and licence_resource.startswith(spdx_prefix): + licence = licence_resource.removeprefix(spdx_prefix) + + features: dict[str, dict[str, str]] = {kind: {} for kind in FEATURE_KINDS} + for kind in FEATURE_KINDS: + prefix = FEATURE_PREFIX[kind] + for supported in project.findall(f"{prefix}:SupportedFeature", DOAP_NS): + name = supported.get("feature") + status = supported.get("status") + if name in KNOWN_FEATURES[kind] and status in FEATURE_STATUSES: + features[kind][name] = status + + room_uri = text("matrix:room") + + return { + "name": text("doap:name"), + "shortdesc": text("doap:shortdesc"), + "licence": licence, + "room": parse_matrix_room_uri(room_uri) if room_uri else None, + "features": features, + } + + +def fetch_and_parse_doap( + url: str, cache_key: str, cache_dir: Path +) -> tuple[TomlTable | None, list[str]]: + """Returns (parsed data or None, issues) - issues covers both fetch/parse + failures and non-fatal warnings (e.g. an unrecognised feature name), so + callers can always report them regardless of whether parsing succeeded.""" + body = fetch_doap_bytes(url, cache_key, cache_dir) + if body is None: + return None, [f"could not fetch {url}"] + try: + root = ET.fromstring(body) + except ET.ParseError as error: + return None, [f"malformed XML: {error}"] + errors = validate_doap(root) + if errors: + return None, errors + return parse_doap(root), doap_warnings(root) + + +def validate_doap_url(url: str) -> list[str]: + """Used by pr-bot.yml to give contributors immediate feedback on doap_url.""" + status, body = http_get(url) + if status != 200: + return [f"could not fetch {url} (HTTP {status})"] + try: + root = ET.fromstring(body) + except ET.ParseError as error: + return [f"malformed XML: {error}"] + return validate_doap(root) + doap_warnings(root) + + +# Change detection + + +def flat_features(fields: TomlTable) -> dict[str, str]: + merged: dict[str, str] = {} + for sub_table in fields.get("features", {}).values(): + if isinstance(sub_table, dict): + merged.update(sub_table) + return merged + + +def collect_changes( + fields: TomlTable, + config: CategoryConfig, + cache_key: str, + github_token: str | None, + cache_dir: Path, +) -> EntryReport: + report = EntryReport(location=cache_key) + + repo_url = fields.get(config.repo_field) + if repo_url: + if ( + is_archived(repo_url, github_token) + and fields.get(config.maturity_field) != "Obsolete" + ): + report.edits.append( + FieldEdit( + config.maturity_field, + fields.get(config.maturity_field), + "Obsolete", + "upstream repository is archived", + ) + ) + if is_dead_link(repo_url) is True: + report.findings.append(f"repo link is dead (404/410): {repo_url}") + + website = fields.get("website") or fields.get("home") + if website and is_dead_link(website) is True: + report.findings.append(f"website link is dead (404/410): {website}") + + doap_url = fields.get(config.doap_field) + if not doap_url: + return report + + doap, issues = fetch_and_parse_doap(doap_url, cache_key, cache_dir) + report.findings.extend(issues) + if doap is None: + return report + + if doap["licence"] and doap["licence"] != fields.get(config.licence_field): + report.edits.append( + FieldEdit( + config.licence_field, + fields.get(config.licence_field), + doap["licence"], + f"declared in DOAP file ({doap_url})", + ) + ) + if doap["room"] and doap["room"] != fields.get(config.room_field): + report.edits.append( + FieldEdit( + config.room_field, + fields.get(config.room_field), + doap["room"], + f"declared in DOAP file ({doap_url})", + ) + ) + if doap["shortdesc"]: + current_description = fields.get("description") + if ( + current_description is None + or current_description.strip() != doap["shortdesc"].strip() + ): + report.findings.append( + f"DOAP shortdesc differs, review manually: {doap['shortdesc']!r}" + ) + for feature_name, status in doap["features"][config.kind].items(): + if flat_features(fields).get(feature_name) != status: + report.findings.append( + f"DOAP declares feature {feature_name}={status} " + f"(review and update [extra.features] by hand)" + ) + + return report + + +def apply_edits(text: str, edits: list[FieldEdit]) -> str: + for edit in edits: + pattern = re.compile(rf'(?m)^({re.escape(edit.field)}\s*=\s*)"[^"]*"') + text, count = pattern.subn(lambda m: f'{m.group(1)}"{edit.new}"', text, count=1) + if count == 0: + print( + f" ! could not locate field {edit.field!r} to apply edit", + file=sys.stderr, + ) + return text + + +# Clients (frontmatter-per-file) + + +def split_frontmatter(text: str) -> tuple[str, str, str] | None: + lines = text.split("\n") + if not lines or lines[0] != "+++": + return None + try: + end = lines.index("+++", 1) + except ValueError: + return None + pre = "+++\n" + body = "\n".join(lines[1:end]) + post = "\n" + "\n".join(lines[end:]) + return pre, body, post + + +def process_clients( + github_token: str | None, cache_dir: Path, dry_run: bool +) -> list[EntryReport]: + reports = [] + for path in sorted(CLIENTS_DIR.glob("*.md")): + if path.name == "_index.md": + continue + text = path.read_text(encoding="utf-8") + split = split_frontmatter(text) + if split is None: + continue + pre, body, post = split + fields = tomllib.loads(body).get("extra", {}) + report = collect_changes( + fields, CLIENT_CONFIG, path.name, github_token, cache_dir + ) + if report.edits and not dry_run: + path.write_text( + pre + apply_edits(body, report.edits) + post, encoding="utf-8" + ) + reports.append(report) + return reports + + +# Servers (shared TOML array of tables) + + +def split_toml_array(text: str, table: str) -> tuple[str, list[str]]: + marker = f"[[{table}]]" + parts = text.split(marker) + return parts[0], [marker + part for part in parts[1:]] + + +def process_servers( + github_token: str | None, cache_dir: Path, dry_run: bool +) -> list[EntryReport]: + text = SERVERS_FILE.read_text(encoding="utf-8") + header, blocks = split_toml_array(text, "servers") + + reports = [] + new_blocks = [] + changed = False + for block in blocks: + entry = tomllib.loads(block)["servers"][0] + name = entry.get("name", "unknown") + report = collect_changes(entry, SERVER_CONFIG, name, github_token, cache_dir) + if report.edits: + block = apply_edits(block, report.edits) + changed = True + new_blocks.append(block) + reports.append(report) + + if changed and not dry_run: + SERVERS_FILE.write_text(header + "".join(new_blocks), encoding="utf-8") + return reports + + +# Reporting + + +def render_findings(reports: list[EntryReport]) -> str: + active = [r for r in reports if r.edits or r.findings] + if not active: + return "No changes or findings this run." + + lines = [] + for report in active: + lines.append(f"#### {report.location}") + for edit in report.edits: + lines.append( + f"- `{edit.field}`: `{edit.old}` -> `{edit.new}` ({edit.reason})" + ) + for finding in report.findings: + lines.append(f"- [ ] {finding}") + lines.append("") + return "\n".join(lines) + + +def render_issue_body(reports: list[EntryReport]) -> str: + """Body for a standing tracking issue, updated in place every run. Findings + with no accompanying edit produce no file diff and so never surface via the + PR - this issue is what actually surfaces those (e.g. a link that's been + dead for months, or a doap_url that started failing after being valid at + setup).""" + active = [r for r in reports if r.findings] + lines = [ + "Automatically maintained by the weekly ecosystem data refresh - " + "issues found with no proposed fix (e.g. a persistently dead link, or " + "a `doap_url` that broke after being valid at setup). Updated in place " + "every run.", + "", + ] + if not active: + lines.append("Nothing outstanding right now.") + return "\n".join(lines) + for report in active: + lines.append(f"### {report.location}") + lines.extend(f"- [ ] {finding}" for finding in report.findings) + lines.append("") + return "\n".join(lines) + + +def render_pr_body(reports: list[EntryReport]) -> str: + """Follows .github/pull_request_template.md so this isn't auto-drafted by pr-bot.yml.""" + return f"""### Description + +Automated weekly refresh of ecosystem client/server data: repo archived status, dead +links, and any changes declared in a maintainer's `doap_url` file +(see `doc/ecosystem-doap-extension.md`). Subjective fields (features, good_for, +maintainer, etc.) are never changed by this job — please review each item below. + +{render_findings(reports)} + +### Related issues + +Part of the automation proposed in #2010. + +### Role + +Automated bot run (Website & Content WG infrastructure). + +### Timeline + +No deadline, review whenever convenient. + +### Signoff + +N/A, automated commit. +""" + + +# Entry point + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--dry-run", action="store_true", help="print diffs, don't write files" + ) + parser.add_argument("--cache-dir", type=Path, default=Path(".cache/doap")) + parser.add_argument("--pr-body-file", type=Path) + parser.add_argument("--issue-body-file", type=Path) + parser.add_argument( + "--validate-doap-url", help="fetch and validate a single DOAP URL, then exit" + ) + args = parser.parse_args() + + if args.validate_doap_url: + errors = validate_doap_url(args.validate_doap_url) + for error in errors: + print(error, file=sys.stderr) + return 1 if errors else 0 + + args.cache_dir.mkdir(parents=True, exist_ok=True) + github_token = os.environ.get("GITHUB_TOKEN") + + reports = process_clients(github_token, args.cache_dir, args.dry_run) + reports += process_servers(github_token, args.cache_dir, args.dry_run) + + print(render_findings(reports)) + if args.pr_body_file: + args.pr_body_file.write_text(render_pr_body(reports), encoding="utf-8") + if args.issue_body_file: + args.issue_body_file.write_text(render_issue_body(reports), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/ecosystem-entry-check.yml b/.github/workflows/ecosystem-entry-check.yml new file mode 100644 index 0000000000..cb43154e5c --- /dev/null +++ b/.github/workflows/ecosystem-entry-check.yml @@ -0,0 +1,45 @@ +name: Ecosystem Entry Check + +on: + pull_request_target: # zizmor: ignore[dangerous-triggers] -- read-only: inspects the PR diff via `gh api`, never checks out or runs code from the fork + types: [opened, reopened, ready_for_review, synchronize] + paths: + - "content/ecosystem/clients/**" + - "content/ecosystem/servers/servers.toml" + +permissions: {} + +concurrency: + group: ecosystem-entry-check-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + check: + name: Check new/changed ecosystem entries + runs-on: ubuntu-latest + permissions: + pull-requests: write # comment on the PR + contents: read # checkout our own script from the base branch + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + # No `ref:` here on purpose - always our own trusted main, never the PR head. + + - name: Check if PR is bot-authored + id: author + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.number }} + run: | + is_bot="$(gh pr view "$PR_NUMBER" --json author --jq '.author.is_bot')" + echo "is_bot=$is_bot" >> "$GITHUB_OUTPUT" + + - name: Check entries and comment + if: steps.author.outputs.is_bot != 'true' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: python3 .github/scripts/check_ecosystem_pr.py diff --git a/.github/workflows/refresh-ecosystem-data.yml b/.github/workflows/refresh-ecosystem-data.yml new file mode 100644 index 0000000000..581a744ac8 --- /dev/null +++ b/.github/workflows/refresh-ecosystem-data.yml @@ -0,0 +1,105 @@ +name: Refresh Ecosystem Data + +on: + schedule: + - cron: "0 6 * * 1" # every Monday at 06:00 UTC + workflow_dispatch: + inputs: + dry_run: + description: "Print proposed diffs without opening/updating a PR" + type: boolean + default: false + +permissions: {} + +# Never runs the writer job twice at once - avoids two runs racing to push to the same +# branch. +concurrency: + group: refresh-ecosystem-data + cancel-in-progress: false + +jobs: + refresh: + name: Refresh client/server data and open a PR + runs-on: ubuntu-latest + permissions: + contents: write # push the automated/refresh-ecosystem-data branch + pull-requests: write # open/update the PR from that branch + issues: write # create/update the standing outstanding-issues tracking issue + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: true # needed for the plain `git push` below + + # Bridges transient outages on a maintainer's site between runs. Deliberate + # removals (404/410) are never served from this cache - see + # fetch_doap_bytes() in the script - so this can't be used to route around a + # DOAP file being taken down for moderation or any other reason. + - name: Restore DOAP fetch cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .cache/doap + key: doap-cache-${{ github.run_id }} + restore-keys: doap-cache- + + - name: Run refresh script + env: + GITHUB_TOKEN: ${{ github.token }} + RUNNER_TEMP: ${{ runner.temp }} + DRY_RUN: ${{ inputs.dry_run == true && '--dry-run' || '' }} + run: | + python3 .github/scripts/refresh_ecosystem_data.py $DRY_RUN \ + --pr-body-file "${RUNNER_TEMP}/pr-body.md" \ + --issue-body-file "${RUNNER_TEMP}/issue-body.md" + + # No-op if there's no diff; reuses the fixed branch/PR on repeat runs. + - name: Open or update pull request + if: ${{ inputs.dry_run != true }} + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + RUNNER_TEMP: ${{ runner.temp }} + BRANCH: automated/refresh-ecosystem-data + TITLE: "Automated ecosystem client/server data refresh" + run: | + set -euo pipefail + + if git diff --quiet -- content/ecosystem/clients content/ecosystem/servers/servers.toml; then + echo "No changes, nothing to do." + exit 0 + fi + + base="$(git branch --show-current)" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -b "$BRANCH" + git add content/ecosystem/clients content/ecosystem/servers/servers.toml + git commit -s -m "Automated: refresh ecosystem client/server data" + git push --force origin "$BRANCH" + + number="$(gh pr list -R "$REPO" --head "$BRANCH" --state open --json number --jq '.[0].number')" + if [ -n "$number" ]; then + gh pr edit "$number" -R "$REPO" --title "$TITLE" --body-file "${RUNNER_TEMP}/pr-body.md" + else + gh pr create -R "$REPO" --title "$TITLE" --body-file "${RUNNER_TEMP}/pr-body.md" \ + --label ecosystem --base "$base" --head "$BRANCH" + fi + + # Standing issue for findings with no accompanying edit (the PR step above + # only runs on an actual diff). Kept up to date in place; never closed + # here - pinning/locking is done manually. + - name: Update the outstanding-issues tracking issue + if: ${{ inputs.dry_run != true }} + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + ISSUE_TITLE: "Ecosystem data refresh: outstanding issues" + ISSUE_BODY_FILE: ${{ runner.temp }}/issue-body.md + run: | + number="$(gh issue list -R "$REPO" --search "in:title \"$ISSUE_TITLE\"" --json number --jq '.[0].number')" + if [ -n "$number" ]; then + gh issue edit "$number" -R "$REPO" --body-file "$ISSUE_BODY_FILE" + else + gh issue create -R "$REPO" --title "$ISSUE_TITLE" --body-file "$ISSUE_BODY_FILE" --label ecosystem + fi diff --git a/.gitignore b/.gitignore index df13bc12ac..70aca9e72a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ .DS_Store gatsby .vscode +__pycache__/ +*.pyc +.cache/ diff --git a/CONTENT.md b/CONTENT.md index 2a613ce0eb..3c5dcab97d 100644 --- a/CONTENT.md +++ b/CONTENT.md @@ -135,6 +135,25 @@ _Due to restrictions on the third-party consumers it is mandatory that we use PN All of the ecosystem projects information are in subdirectories of [`/content/ecosystem`](https://github.com/matrix-org/matrix.org/tree/main/content/ecosystem/). +### Automated data refresh + +Clients and servers are checked weekly by a scheduled job +(`.github/workflows/refresh-ecosystem-data.yml`): it flags dead repo/website links (any +host), and proposes bumping `maturity` to `Obsolete` when a linked repo gets archived. +The archived-repo check only understands GitHub and GitLab - repos on other forges or +VCS systems are simply skipped for that specific check and +still get the dead-link check. This needs nothing from you beyond the +`repo`/`repository` field you'd add anyway. + +If you want your listing kept fully in sync, including feature support, publish a small +[DOAP](https://github.com/ewilderj/doap) file describing your project and set the optional +`doap_url` field to its URL — see +[`doc/ecosystem-doap-extension.md`](https://github.com/matrix-org/matrix.org/blob/main/doc/ecosystem-doap-extension.md) +for the format. + +Any changes will be proposed as an automated PR requiring a human review. +The COC and our repository rules still apply to this content. + ### Clients Matrix clients are listed in [`/content/ecosystem/clients`](https://github.com/matrix-org/matrix.org/tree/main/content/ecosystem/clients). Every client has its individual page, so every client is represented by a markdown file. Most of the information is living in the _frontmatter_, between the two `+++` rows in a `.md` file. @@ -152,6 +171,9 @@ maturity = "PICK ONE Stable OR Beta OR Alpha OR Obsolete" repo = "https://github.com/example-org/example-repo" matrix_room = "#your-matrix-room:example.com" licence = "PICK ONE identifier from https://spdx.org/licenses/" +# Optional: a DOAP file you host and control, for automated data refresh. +# See doc/ecosystem-doap-extension.md. +# doap_url = "https://raw.githubusercontent.com/example-org/example-repo/main/matrix.doap" featured = false # Used with featured = true to have a fixed order. # featured_order = 1 @@ -201,6 +223,7 @@ Supercharge your communications with Example Client. - For `extra.features`, see the descriptions in [clients.html](/templates/macros/clients.html) - All of the properties under `extra.packages` are optional: only add the installation methods your project supports! - In case your option is not available please let us know by opening an issue. +- `doap_url` is optional. See "Automated data refresh" above. ### Bridges @@ -255,6 +278,9 @@ licence = "PICK ONE identifier from https://spdx.org/licenses/" repository = "https://github.com/example-org/example-repo" website = "https://mymatrixserver.dev" room = "#your-matrix-room:example.com" +# Optional: a DOAP file you host and control, for automated data refresh. +# See doc/ecosystem-doap-extension.md. +# doap_url = "https://raw.githubusercontent.com/example-org/example-repo/main/matrix.doap" ``` ### Integrations diff --git a/doc/ecosystem-doap-extension.md b/doc/ecosystem-doap-extension.md new file mode 100644 index 0000000000..c12d50387f --- /dev/null +++ b/doc/ecosystem-doap-extension.md @@ -0,0 +1,138 @@ +# Matrix ecosystem DOAP extension + +This is the file format read by the ecosystem data refresh automation +(`.github/scripts/refresh_ecosystem_data.py`). It lets a +client or server maintainer publish a small file describing their project, which we fetch +and propose as an update to their `content/ecosystem/{clients,servers}` entry. Publishing +one is entirely optional. + +## How to use it + +1. Write a DOAP file describing your project (template below). +2. Host it anywhere reachable over plain HTTPS — your own site, or a raw file link inside + your git repo (e.g. `https://raw.githubusercontent.com/you/project/main/matrix.doap`). +3. Set `doap_url` to that URL: + - clients: `extra.doap_url = "https://..."` in your entry's frontmatter + - servers: `doap_url = "https://..."` in your `[[servers]]` entry in `servers.toml` + +A weekly job fetches it, and any changes are proposed as a diff in a PR for a human to +review - nothing is auto-merged, and nothing is auto-fetched unless you set `doap_url`. + +## Base vocabulary: DOAP + +We read standard [DOAP](https://github.com/ewilderj/doap) (`http://usefulinc.com/ns/doap#`) +fields: + +| DOAP field | Maps to | +| --------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | +| `doap:name` | `title` | +| `doap:shortdesc` | the entry's short body text | +| `doap:homepage` | `website`/`home` | +| `doap:repository` / `doap:GitRepository` / `doap:SVNRepository` → `doap:location` | `repo` (clients) / `repository` (servers) | +| `doap:license` | `licence` — must be an `rdf:resource` under `https://spdx.org/licenses/`; anything else is ignored | +| `doap:maintainer` / `foaf:Person` / `foaf:name` | `maintainer` | + +Fields we don't read are simply ignored. You can include whatever other standard DOAP +data you like. + +## Extension namespaces + +Three namespaces, defined by us, for the parts DOAP has no vocabulary for: + +| Prefix | Namespace | Used for | +| --------------- | ----------------------------------------- | ------------------------------- | +| `matrix` | `https://matrix.org/ns/ecosystem#` | shared elements (`matrix:room`) | +| `matrix_client` | `https://matrix.org/ns/ecosystem/client#` | client feature support | +| `matrix_server` | `https://matrix.org/ns/ecosystem/server#` | server feature support | + +### `matrix:room` + +A [`matrix:` URI](https://spec.matrix.org/v1.18/appendices/#matrix-uri-scheme) — either a +room alias (`r`) or a room ID (`roomid`): + +```xml +matrix:r/your-room:example.com +``` + +```xml +matrix:roomid/opaqueid:example.com +``` + +Maps to `matrix_room` (clients) / `room` (servers), converted back to their sigil form +(`#your-room:example.com` / `!opaqueid:example.com`). Anything that isn't a valid `r` or +`roomid` `matrix:` URI is flagged as a warning when your PR is opened, and ignored. + +### `matrix_client:SupportedFeature` + +One element per feature you have an opinion on, as a child of `doap:Project`: + +```xml + +``` + +- `feature` — one of: `e2ee`, `spaces`, `voip_1to1`, `threads`, `sso`, `voip_jitsi`, + `multi_account`, `multi_language`, `oauth`, `invisible_crypto`, `voip_matrixrtc`, + `sliding_sync` (kept in sync with `[extra.features]` in CONTENT.md). Anything else is + flagged as a likely typo when your PR is opened, and ignored. +- `status` — one of `supported`, `partial`, `unsupported`. + +Maps to `[extra.features]`. + +### `matrix_server:SupportedFeature` + +Same shape as `matrix_client:SupportedFeature`, reserved for when servers get a published +feature matrix of their own. Accepted today, but not yet applied anywhere (there's +currently no known server feature name, so anything here is flagged as unrecognised). + +## Validation + +A `doap_url` you set is checked when your PR is opened (see `ecosystem-entry-check.yml`) +and on every weekly run (see `refresh-ecosystem-data.yml`): + +**Hard errors** - the whole file is skipped (logged, not applied) if any of these fail: + +- must be well-formed XML +- must contain exactly one `doap:Project` +- every `matrix_client:`/`matrix_server:SupportedFeature` must have a `feature` and a + `status`, and `status` must be one of the three values above + +**Warnings** - reported, but only that one piece of data is ignored, the rest of the file +still applies: + +- `matrix:room` that isn't a valid room-alias/room-id `matrix:` URI +- a `feature` name we don't recognise (likely a typo) + +Either way, issues are also visible in the standing "Ecosystem data refresh: outstanding +issues" tracking issue if they persist across weekly runs. + +## Full example + +```xml + + + + Example Client + Supercharge your communications with Example Client. + + + + + + + + + + Your name or org + + + matrix:r/example-client:example.com + + + + + +```