feat(ingest): pure classify_url for git vs docs vs unsupported

src/readme2demo/ingest.py

@@ -22,7 +22,7 @@
 from pathlib import Path
 
 from readme2demo import llm
-from readme2demo.types import Plan
+from readme2demo.types import Plan, UrlVerdict
 
 _URL_RE = re.compile(
     r"^https://(?:github|gitlab)\.com/"  # allowed hosts only (MVP)
@@ -51,6 +51,80 @@ class IngestError(RuntimeError):
     """Raised when cloning or planning cannot proceed (bad URL, git failure)."""
 
 
+
+# Hosts we treat as cloneable git forges in the MVP. Keep in sync with _URL_RE.
+_GIT_HOSTS = ("github.com", "gitlab.com")
+# Path segments that indicate a browser deep-link into a repo (not extra GitLab groups).
+_GIT_DEEP_LINK_MARKERS = frozenset({"tree", "blob", "commit", "pulls", "pull", "issues", "wiki", "releases", "actions", "settings", "projects", "network", "security", "pulse", "graphs"})
+
+
+def classify_url(url: str) -> UrlVerdict:
+    """Classify *url* as a git repository, hosted docs page, or unsupported input.
+
+    Pure string logic only — no network, filesystem, or subprocess. Intended as
+    the vocabulary for docs-site ingestion (#67); not yet wired into the pipeline.
+    """
+    raw = (url or "").strip()
+    if not raw:
+        return UrlVerdict(kind="unsupported", reason="empty URL")
+
+    # Local paths / file URLs — unsupported here; local acceptance is #74.
+    if raw.startswith("file:") or raw.startswith("/") or raw.startswith("."):
+        return UrlVerdict(kind="unsupported", reason="local path is not supported")
+
+    # SSH remotes are not accepted by the MVP clone path.
+    if raw.startswith("git@") or raw.startswith("ssh://"):
+        return UrlVerdict(kind="unsupported", reason="ssh remotes are not supported — use https")
+
+    # Reject embedded-URL smuggling early (must never be classified as git).
+    lower = raw.casefold()
+    if lower.count("://") > 1 or "https://" in lower[8:]:
+        return UrlVerdict(kind="unsupported", reason="URL contains an embedded URL and is not a plain git host path")
+
+    from urllib.parse import urlparse
+
+    parsed = urlparse(raw)
+    scheme = (parsed.scheme or "").casefold()
+    if scheme == "http":
+        return UrlVerdict(kind="unsupported", reason="plain http is not supported — use https")
+    if scheme != "https":
+        return UrlVerdict(kind="unsupported", reason=f"unsupported scheme {parsed.scheme!r}")
+
+    host = (parsed.hostname or "").casefold()
+    if host.startswith("www."):
+        host = host[4:]
+
+    # GitHub Pages sites are docs, not cloneable repos.
+    if host.endswith(".github.io"):
+        return UrlVerdict(kind="docs", reason="GitHub Pages site")
+
+    path = (parsed.path or "").strip("/")
+    # Drop trailing .git for matching
+    path_no_git = path[:-4] if path.endswith(".git") else path
+    segments = [s for s in path_no_git.split("/") if s]
+
+    if host in _GIT_HOSTS:
+        if len(segments) < 2:
+            return UrlVerdict(
+                kind="unsupported",
+                reason="no owner/repo path segment",
+            )
+        # Reduce deep links to owner/repo root. GitLab subgroups keep extra
+        # segments until a known deep-link marker appears.
+        owner_repo: list[str] = []
+        for i, seg in enumerate(segments):
+            if seg in _GIT_DEEP_LINK_MARKERS and i >= 2:
+                break
+            owner_repo.append(seg)
+        if len(owner_repo) < 2:
+            return UrlVerdict(kind="unsupported", reason="no owner/repo path segment")
+        repo_url = f"https://{host}/{'/'.join(owner_repo)}"
+        return UrlVerdict(kind="git", repo_url=repo_url, reason="cloneable repository URL")
+
+    # Any other https URL is treated as a potential docs page for this slice.
+    return UrlVerdict(kind="docs", reason="non-git https URL treated as docs page")
+
+
 def clone_repo(repo_url: str, dest: Path, timeout: int = 300) -> str:
     """Shallow-clone ``repo_url`` into ``dest`` and return the HEAD commit sha.
 

src/readme2demo/types.py

@@ -30,6 +30,7 @@
 
 Phase = Literal["explore", "setup", "demo", "fix", "unknown"]
 Outcome = Literal["success", "blocked", "failed"]
+SourceKind = Literal["git", "docs", "unsupported"]
 
 # Sentinel markers the agent prompt instructs the agent to print.
 SUCCESS_MARKER = "R2D_SUCCESS"
@@ -41,6 +42,18 @@
 ADJUSTED_MARKER = "ADJUSTED_SUCCESS:"
 
 
+class UrlVerdict(BaseModel):
+    """Result of pure URL classification for future docs-site ingestion (#67).
+
+    ``kind`` is the coarse class; ``repo_url`` is a cloneable repository root when
+    ``kind == "git"`` (deep links are reduced to that root). ``reason`` is a short
+    human-readable explanation suitable for error messages.
+    """
+
+    kind: SourceKind
+    repo_url: Optional[str] = None
+    reason: str = ""
+
 class SuccessCriteria(BaseModel):
     """Machine-checkable definition of 'the quickstart works'.
 

tests/test_ingest.py

@@ -8,6 +8,7 @@
 from readme2demo import llm
 from readme2demo.ingest import (
     IngestError,
+    classify_url,
     clone_repo,
     collect_docs,
     collect_inventory,
@@ -385,3 +386,61 @@ def boom_clone(*a, **k):
     monkeypatch.setattr(ingest_mod, "clone_repo", boom_clone)
     with pytest.raises(IngestError, match="Nothing to ingest"):
         ingest_mod.ingest(None, tmp_path / "run", "m", guide_file=None)
+
+# -- classify_url --------------------------------------------------------------
+
+
+
+@pytest.mark.parametrize(
+    "url,kind,repo_url",
+    [
+        ("https://github.com/owner/repo", "git", "https://github.com/owner/repo"),
+        ("https://github.com/owner/repo/", "git", "https://github.com/owner/repo"),
+        ("https://github.com/owner/repo.git", "git", "https://github.com/owner/repo"),
+        ("https://gitlab.com/group/subgroup/repo", "git", "https://gitlab.com/group/subgroup/repo"),
+        (
+            "https://github.com/owner/repo/tree/main/examples",
+            "git",
+            "https://github.com/owner/repo",
+        ),
+        (
+            "https://github.com/owner/repo/blob/main/README.md",
+            "git",
+            "https://github.com/owner/repo",
+        ),
+        ("https://docs.example.com/quickstart", "docs", None),
+        ("https://example.com", "docs", None),
+        ("https://owner.github.io/project/", "docs", None),
+        ("https://github.com/owner", "unsupported", None),
+        ("https://github.com/", "unsupported", None),
+        ("http://github.com/owner/repo", "unsupported", None),
+        ("git@github.com:owner/repo.git", "unsupported", None),
+        ("ssh://git@github.com/owner/repo.git", "unsupported", None),
+        ("/local/path/to/repo", "unsupported", None),
+        ("file:///tmp/repo", "unsupported", None),
+        ("", "unsupported", None),
+        ("https://evil.example/https://github.com/owner/repo", "unsupported", None),
+    ],
+)
+def test_classify_url_table(url, kind, repo_url):
+    """Table-driven classification for the #194 policy floor."""
+    verdict = classify_url(url)
+    assert verdict.kind == kind
+    assert verdict.repo_url == repo_url
+    assert isinstance(verdict.reason, str)
+
+
+def test_classify_url_evil_embedded_never_git():
+    v = classify_url("https://evil.example/https://github.com/owner/repo")
+    assert v.kind != "git"
+
+
+def test_classify_url_deep_links_reduce_to_root():
+    for url in (
+        "https://github.com/acme/tool/tree/main/examples",
+        "https://github.com/acme/tool/blob/main/README.md",
+    ):
+        v = classify_url(url)
+        assert v.kind == "git"
+        assert v.repo_url == "https://github.com/acme/tool"
+