From 5ab346ab2618e803fb132b46a7b826b8c4346728 Mon Sep 17 00:00:00 2001 From: Mohammed Anas Nathani Date: Thu, 23 Jul 2026 01:25:47 +0530 Subject: [PATCH] feat(ingest): add pure classify_url for git vs docs vs unsupported MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce SourceKind/UrlVerdict and a side-effect-free classifier that recognizes GitHub/GitLab roots and deep links, treats other https as docs, and rejects ssh/http/local/embedded-URL inputs. Not wired into the pipeline yet — vocabulary for #67. Closes #194 --- src/readme2demo/ingest.py | 76 ++++++++++++++++++++++++++++++++++++++- src/readme2demo/types.py | 13 +++++++ tests/test_ingest.py | 59 ++++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+), 1 deletion(-) diff --git a/src/readme2demo/ingest.py b/src/readme2demo/ingest.py index 8163961..9e96abf 100644 --- a/src/readme2demo/ingest.py +++ b/src/readme2demo/ingest.py @@ -22,7 +22,7 @@ from pathlib import Path from readme2demo import llm -from readme2demo.types import Plan +from readme2demo.types import Plan, UrlVerdict _URL_RE = re.compile( r"^https://(?:github|gitlab)\.com/" # allowed hosts only (MVP) @@ -51,6 +51,80 @@ class IngestError(RuntimeError): """Raised when cloning or planning cannot proceed (bad URL, git failure).""" + +# Hosts we treat as cloneable git forges in the MVP. Keep in sync with _URL_RE. +_GIT_HOSTS = ("github.com", "gitlab.com") +# Path segments that indicate a browser deep-link into a repo (not extra GitLab groups). +_GIT_DEEP_LINK_MARKERS = frozenset({"tree", "blob", "commit", "pulls", "pull", "issues", "wiki", "releases", "actions", "settings", "projects", "network", "security", "pulse", "graphs"}) + + +def classify_url(url: str) -> UrlVerdict: + """Classify *url* as a git repository, hosted docs page, or unsupported input. + + Pure string logic only — no network, filesystem, or subprocess. Intended as + the vocabulary for docs-site ingestion (#67); not yet wired into the pipeline. + """ + raw = (url or "").strip() + if not raw: + return UrlVerdict(kind="unsupported", reason="empty URL") + + # Local paths / file URLs — unsupported here; local acceptance is #74. + if raw.startswith("file:") or raw.startswith("/") or raw.startswith("."): + return UrlVerdict(kind="unsupported", reason="local path is not supported") + + # SSH remotes are not accepted by the MVP clone path. + if raw.startswith("git@") or raw.startswith("ssh://"): + return UrlVerdict(kind="unsupported", reason="ssh remotes are not supported — use https") + + # Reject embedded-URL smuggling early (must never be classified as git). + lower = raw.casefold() + if lower.count("://") > 1 or "https://" in lower[8:]: + return UrlVerdict(kind="unsupported", reason="URL contains an embedded URL and is not a plain git host path") + + from urllib.parse import urlparse + + parsed = urlparse(raw) + scheme = (parsed.scheme or "").casefold() + if scheme == "http": + return UrlVerdict(kind="unsupported", reason="plain http is not supported — use https") + if scheme != "https": + return UrlVerdict(kind="unsupported", reason=f"unsupported scheme {parsed.scheme!r}") + + host = (parsed.hostname or "").casefold() + if host.startswith("www."): + host = host[4:] + + # GitHub Pages sites are docs, not cloneable repos. + if host.endswith(".github.io"): + return UrlVerdict(kind="docs", reason="GitHub Pages site") + + path = (parsed.path or "").strip("/") + # Drop trailing .git for matching + path_no_git = path[:-4] if path.endswith(".git") else path + segments = [s for s in path_no_git.split("/") if s] + + if host in _GIT_HOSTS: + if len(segments) < 2: + return UrlVerdict( + kind="unsupported", + reason="no owner/repo path segment", + ) + # Reduce deep links to owner/repo root. GitLab subgroups keep extra + # segments until a known deep-link marker appears. + owner_repo: list[str] = [] + for i, seg in enumerate(segments): + if seg in _GIT_DEEP_LINK_MARKERS and i >= 2: + break + owner_repo.append(seg) + if len(owner_repo) < 2: + return UrlVerdict(kind="unsupported", reason="no owner/repo path segment") + repo_url = f"https://{host}/{'/'.join(owner_repo)}" + return UrlVerdict(kind="git", repo_url=repo_url, reason="cloneable repository URL") + + # Any other https URL is treated as a potential docs page for this slice. + return UrlVerdict(kind="docs", reason="non-git https URL treated as docs page") + + def clone_repo(repo_url: str, dest: Path, timeout: int = 300) -> str: """Shallow-clone ``repo_url`` into ``dest`` and return the HEAD commit sha. diff --git a/src/readme2demo/types.py b/src/readme2demo/types.py index 6394e27..0b14a7a 100644 --- a/src/readme2demo/types.py +++ b/src/readme2demo/types.py @@ -30,6 +30,7 @@ Phase = Literal["explore", "setup", "demo", "fix", "unknown"] Outcome = Literal["success", "blocked", "failed"] +SourceKind = Literal["git", "docs", "unsupported"] # Sentinel markers the agent prompt instructs the agent to print. SUCCESS_MARKER = "R2D_SUCCESS" @@ -41,6 +42,18 @@ ADJUSTED_MARKER = "ADJUSTED_SUCCESS:" +class UrlVerdict(BaseModel): + """Result of pure URL classification for future docs-site ingestion (#67). + + ``kind`` is the coarse class; ``repo_url`` is a cloneable repository root when + ``kind == "git"`` (deep links are reduced to that root). ``reason`` is a short + human-readable explanation suitable for error messages. + """ + + kind: SourceKind + repo_url: Optional[str] = None + reason: str = "" + class SuccessCriteria(BaseModel): """Machine-checkable definition of 'the quickstart works'. diff --git a/tests/test_ingest.py b/tests/test_ingest.py index 75dfcb3..10b6858 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -8,6 +8,7 @@ from readme2demo import llm from readme2demo.ingest import ( IngestError, + classify_url, clone_repo, collect_docs, collect_inventory, @@ -385,3 +386,61 @@ def boom_clone(*a, **k): monkeypatch.setattr(ingest_mod, "clone_repo", boom_clone) with pytest.raises(IngestError, match="Nothing to ingest"): ingest_mod.ingest(None, tmp_path / "run", "m", guide_file=None) + +# -- classify_url -------------------------------------------------------------- + + + +@pytest.mark.parametrize( + "url,kind,repo_url", + [ + ("https://github.com/owner/repo", "git", "https://github.com/owner/repo"), + ("https://github.com/owner/repo/", "git", "https://github.com/owner/repo"), + ("https://github.com/owner/repo.git", "git", "https://github.com/owner/repo"), + ("https://gitlab.com/group/subgroup/repo", "git", "https://gitlab.com/group/subgroup/repo"), + ( + "https://github.com/owner/repo/tree/main/examples", + "git", + "https://github.com/owner/repo", + ), + ( + "https://github.com/owner/repo/blob/main/README.md", + "git", + "https://github.com/owner/repo", + ), + ("https://docs.example.com/quickstart", "docs", None), + ("https://example.com", "docs", None), + ("https://owner.github.io/project/", "docs", None), + ("https://github.com/owner", "unsupported", None), + ("https://github.com/", "unsupported", None), + ("http://github.com/owner/repo", "unsupported", None), + ("git@github.com:owner/repo.git", "unsupported", None), + ("ssh://git@github.com/owner/repo.git", "unsupported", None), + ("/local/path/to/repo", "unsupported", None), + ("file:///tmp/repo", "unsupported", None), + ("", "unsupported", None), + ("https://evil.example/https://github.com/owner/repo", "unsupported", None), + ], +) +def test_classify_url_table(url, kind, repo_url): + """Table-driven classification for the #194 policy floor.""" + verdict = classify_url(url) + assert verdict.kind == kind + assert verdict.repo_url == repo_url + assert isinstance(verdict.reason, str) + + +def test_classify_url_evil_embedded_never_git(): + v = classify_url("https://evil.example/https://github.com/owner/repo") + assert v.kind != "git" + + +def test_classify_url_deep_links_reduce_to_root(): + for url in ( + "https://github.com/acme/tool/tree/main/examples", + "https://github.com/acme/tool/blob/main/README.md", + ): + v = classify_url(url) + assert v.kind == "git" + assert v.repo_url == "https://github.com/acme/tool" +