From 3521d47c274ddd9e051d566c5cf050495dc49850 Mon Sep 17 00:00:00 2001 From: Mohammed Anas Nathani Date: Thu, 23 Jul 2026 05:33:25 +0530 Subject: [PATCH] fix(ingest): strip GitHub deep links before git clone Pasted browser URLs like .../tree/main/examples passed validation then died with a raw git error. Recognize GitHub /tree and /blob suffixes, clone the repository root, and warn. GitLab subgroup paths stay intact. Closes #109 --- src/readme2demo/ingest.py | 63 ++++++++++++++++++++++++++++++++++++-- tests/test_ingest.py | 64 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 2 deletions(-) diff --git a/src/readme2demo/ingest.py b/src/readme2demo/ingest.py index 8163961..deeec6e 100644 --- a/src/readme2demo/ingest.py +++ b/src/readme2demo/ingest.py @@ -16,6 +16,8 @@ from __future__ import annotations +import warnings + import json import re import subprocess @@ -50,6 +52,56 @@ class IngestError(RuntimeError): """Raised when cloning or planning cannot proceed (bad URL, git failure).""" +# GitHub deep-link path segments that are NOT GitLab-style subgroups. +_GITHUB_DEEP_LINK_MARKERS = frozenset({ + "tree", "blob", "commit", "pulls", "pull", "issues", "wiki", + "releases", "actions", "settings", "projects", "network", "security", + "pulse", "graphs", "raw", "blame", +}) + + +def _normalize_clone_url(repo_url: str) -> tuple[str, str | None]: + """Return (cloneable_url, optional_user_note) for a validated forge URL. + + GitHub deep links (``/tree/...``, ``/blob/...``) are stripped to the + repository root with a note. Other hosts keep the permissive subgroup + behavior (GitLab subgroups are legitimate path segments). + """ + from urllib.parse import urlparse + + parsed = urlparse(repo_url.strip()) + host = (parsed.hostname or "").casefold() + if host.startswith("www."): + host = host[4:] + path = (parsed.path or "").strip("/") + if path.endswith(".git"): + path = path[:-4] + segments = [s for s in path.split("/") if s] + + if host == "github.com" and len(segments) >= 2: + # owner/repo[/deep-link...] + owner, repo = segments[0], segments[1] + rest = segments[2:] + if rest and rest[0].lower() in _GITHUB_DEEP_LINK_MARKERS: + root = f"https://github.com/{owner}/{repo}" + note = ( + f"Note: {repo_url!r} looks like a GitHub deep link " + f"(/{rest[0]}/...); cloning the repository root {root} instead." + ) + return root, note + if len(segments) > 2 and rest[0].lower() not in _GITHUB_DEEP_LINK_MARKERS: + # Unusual extra path on github without a known marker — still + # clone root with a note rather than raw git failure. + if rest: + root = f"https://github.com/{owner}/{repo}" + note = ( + f"Note: {repo_url!r} has extra path segments after the " + f"repo root; cloning {root} instead." + ) + return root, note + + return repo_url.rstrip("/"), None + def clone_repo(repo_url: str, dest: Path, timeout: int = 300) -> str: """Shallow-clone ``repo_url`` into ``dest`` and return the HEAD commit sha. @@ -63,10 +115,11 @@ def clone_repo(repo_url: str, dest: Path, timeout: int = 300) -> str: f"Invalid repo URL: {repo_url!r} — expected " "https://github.com// or https://gitlab.com//" ) + clone_url, deep_link_note = _normalize_clone_url(repo_url) dest.parent.mkdir(parents=True, exist_ok=True) try: proc = subprocess.run( - ["git", "clone", "--depth", "1", repo_url, str(dest)], + ["git", "clone", "--depth", "1", clone_url, str(dest)], capture_output=True, text=True, timeout=timeout, @@ -77,9 +130,15 @@ def clone_repo(repo_url: str, dest: Path, timeout: int = 300) -> str: except FileNotFoundError as e: raise IngestError("git not found — install git and ensure it is on PATH") from e if proc.returncode != 0: + detail = proc.stderr.strip() or proc.stdout.strip() + hint = "" + if deep_link_note: + hint = f" ({deep_link_note})" raise IngestError( - f"git clone failed ({proc.returncode}): {proc.stderr.strip() or proc.stdout.strip()}" + f"git clone failed ({proc.returncode}): {detail}{hint}" ) + if deep_link_note: + warnings.warn(deep_link_note, UserWarning, stacklevel=2) rev = subprocess.run( ["git", "-C", str(dest), "rev-parse", "HEAD"], capture_output=True, diff --git a/tests/test_ingest.py b/tests/test_ingest.py index 75dfcb3..f9db9ed 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -385,3 +385,67 @@ def boom_clone(*a, **k): monkeypatch.setattr(ingest_mod, "clone_repo", boom_clone) with pytest.raises(IngestError, match="Nothing to ingest"): ingest_mod.ingest(None, tmp_path / "run", "m", guide_file=None) + + +# -- GitHub deep-link normalization (#109) ------------------------------------ + + +def test_normalize_github_deep_link_strips_tree(): + """Regression (#109): browser /tree/ URLs clone the repo root, not die raw.""" + from readme2demo.ingest import _normalize_clone_url + + url, note = _normalize_clone_url( + "https://github.com/owner/repo/tree/main/examples" + ) + assert url == "https://github.com/owner/repo" + assert note is not None + assert "deep link" in note.lower() or "tree" in note.lower() + + +def test_normalize_github_deep_link_strips_blob(): + from readme2demo.ingest import _normalize_clone_url + + url, note = _normalize_clone_url( + "https://github.com/owner/repo/blob/main/README.md" + ) + assert url == "https://github.com/owner/repo" + assert note is not None + + +def test_normalize_gitlab_subgroup_preserved(): + """Regression (#109): GitLab subgroups must still be accepted as-is.""" + from readme2demo.ingest import _normalize_clone_url + + url = "https://gitlab.com/group/subgroup/repo" + out, note = _normalize_clone_url(url) + assert out.rstrip("/") == url + assert note is None + + +def test_clone_repo_uses_root_for_github_deep_link(tmp_path, monkeypatch): + """Regression (#109): clone argv must use the stripped root URL.""" + from readme2demo import ingest as ingest_mod + import subprocess + + captured = {} + + def fake_run(cmd, **kw): + captured.setdefault("cmds", []).append(cmd) + if cmd[:2] == ["git", "clone"]: + dest = Path(cmd[-1]) + dest.mkdir(parents=True, exist_ok=True) + (dest / ".git").mkdir(exist_ok=True) + return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="") + if "rev-parse" in cmd: + return subprocess.CompletedProcess(cmd, 0, stdout="abc1234\n", stderr="") + return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="") + + monkeypatch.setattr(ingest_mod.subprocess, "run", fake_run) + sha = ingest_mod.clone_repo( + "https://github.com/owner/repo/tree/main/examples", + tmp_path / "repo", + ) + assert sha.strip() == "abc1234" + clone_cmd = captured["cmds"][0] + assert "https://github.com/owner/repo" in clone_cmd + assert "tree" not in " ".join(clone_cmd)