diff --git a/.coverage b/.coverage deleted file mode 100644 index 033ae57..0000000 Binary files a/.coverage and /dev/null differ diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index aa6e7b3..0000000 --- a/.coveragerc +++ /dev/null @@ -1,7 +0,0 @@ -[run] -omit = - */site-packages/* - -[report] -exclude_lines = - if __name__ == .__main__.: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..bf7d52d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,86 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + types: [opened, synchronize, reopened] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + name: Lint & type-check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install uv + uses: astral-sh/setup-uv@v3 + + - name: Install dependencies + run: uv pip install --system -e ".[dev,bdd,docs,gitignore,ext]" + + - name: Ruff lint + run: ruff check . + + - name: Ruff format check + run: ruff format --check . + + - name: Mypy + run: mypy --strict src/rglob + + test: + name: Test (${{ matrix.os }} · py${{ matrix.python-version }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["3.11", "3.12", "3.13", "3.14"] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + + - name: Install uv + uses: astral-sh/setup-uv@v3 + + - name: Install dependencies + run: uv pip install --system -e ".[dev,bdd,gitignore,ext]" + + - name: Pytest (with coverage) + shell: bash + run: | + # POSIX-only tests (perm bits, executable kind, uid/gid) are + # skipped on non-Linux hosts; pass --cov-fail-under=0 there so + # both the pytest-cov and the underlying coverage.report + # fail_under gates are disabled. The full-suite gate only + # fires on Ubuntu. + if [ "${{ matrix.os }}" = "ubuntu-latest" ]; then + pytest --cov=rglob --cov-branch --cov-report=xml --cov-fail-under=95 + else + pytest --cov=rglob --cov-branch --cov-report=xml --cov-fail-under=0 + fi + + - name: Behave (BDD) + run: behave + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + files: coverage.xml + flags: ${{ matrix.os }}-py${{ matrix.python-version }} + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..db0263b --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,27 @@ +name: Docs + +on: + push: + branches: [master] + +permissions: + contents: write # to push to gh-pages + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install docs deps + run: pip install -e ".[docs]" + + - name: Deploy MkDocs to gh-pages + run: mkdocs gh-deploy --force --clean diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml deleted file mode 100644 index 50465f4..0000000 --- a/.github/workflows/pylint.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Lint (pylint) - -on: - pull_request: - types: [opened, synchronize, reopened] - -jobs: - pylint: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - cache: 'pip' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt - - - name: Run pylint - run: | - pylint rglob setup.py features diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..18f61e5 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,79 @@ +name: Release + +on: + push: + tags: + - "v*" + +permissions: + contents: write # to create the GitHub Release + id-token: write # for PyPI OIDC trusted publishing + +jobs: + build: + name: Build sdist + wheel + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install hatch + run: pip install hatch + + - name: Build + run: hatch build + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish: + name: Publish to PyPI + needs: build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/rglob + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + + - name: Publish (OIDC, no token) + uses: pypa/gh-action-pypi-publish@release/v1 + + github-release: + name: GitHub Release + needs: publish + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + + - name: Extract changelog slice + id: changelog + run: | + version="${GITHUB_REF_NAME#v}" + awk "/^## \\[$version\\]/{flag=1; next} /^## \\[/{flag=0} flag" CHANGELOG.md > release-notes.md + test -s release-notes.md || echo "Release $version" > release-notes.md + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + body_path: release-notes.md + files: dist/* diff --git a/.gitignore b/.gitignore index d05b513..b632545 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,40 @@ -# Ignore Python-Compiled Files +# Python bytecode + caches *.pyc +__pycache__/ -# Ignore Eclipse/PyDev Project Files +# Editor / IDE .project .pydevproject -.settings +.settings/ +.idea/ +.vscode/ -# Ignore packaging metadata +# Packaging artefacts *.egg-info/ +build/ +dist/ +site/ + +# Virtual environments +.venv/ +venv/ +env/ + +# uv lock file — rglob is a published library, so deps are declared in +# pyproject.toml and the lock file is regenerated per-environment. +uv.lock + +# Test / coverage / type caches +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +.coverage.* +htmlcov/ +coverage.xml +.tox/ +.nox/ + +# OS noise +.DS_Store +Thumbs.db diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..21553ba --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,40 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-yaml + - id: check-toml + - id: check-merge-conflict + - id: end-of-file-fixer + - id: trailing-whitespace + - id: mixed-line-ending + args: [--fix=lf] + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.7.4 + hooks: + - id: ruff + args: [--fix] + - id: ruff-format + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.13.0 + hooks: + - id: mypy + files: ^src/rglob/ + additional_dependencies: [] + + - repo: https://github.com/tox-dev/pyproject-fmt + rev: v2.5.0 + hooks: + - id: pyproject-fmt + + - repo: local + hooks: + - id: pytest-quick + name: pytest (quick, no coverage) + entry: pytest -x --no-cov -q + language: system + types: [python] + pass_filenames: false + stages: [pre-push] diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..6d489b3 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,175 @@ +# AGENTS.md + +Instructions for AI coding assistants (Claude Code, Cursor, Aider, OpenAI +codex-style agents) working in this repository. This is a labor-of-love +hobby package — keep changes scoped, idiomatic, and small. + +## Project at a glance + +- **Name**: `rglob` — lightweight recursive glob helpers for Python. +- **Surface**: `find` / `find_all` (modern), `rglob` / `rglob_` / `lcount` / + `tsize` (legacy), `rglob.agent` (stable agent API), + `kilobytes` / `megabytes` / `gigabytes` / `terabytes` (unit helpers). + CLI: `find`, `grep`, `count`, `lcount`, `tsize`, `stats`, `tree`, `top`, + `dupes`, `describe`, `schema`, `capabilities`, `agent-version`, `mcp`. +- **Python**: 3.11+ (3.10 drops at the 2.0 release — see + [ADR-0002](docs/decisions/0002-python-floor.md)). +- **Build backend**: hatchling. **CLI**: Typer + Rich. +- **Docs**: MkDocs Material → `https://chris-piekarski.github.io/python-rglob/`. + +## Layout + +```text +src/rglob/ # the package + __init__.py # public re-exports + __version__ + rglob.py # find/find_all walker + legacy rglob/lcount/tsize + cli.py # Typer CLI app + _filters.py # size/time/kind parsers and predicates (Phase 5) + _dupes.py # duplicate-detection pipeline (Phase 5) + agent/ # stable agent dataclasses, schemas, MCP server + py.typed # PEP 561 marker +tests/ # primary suite — pytest + hypothesis + syrupy +features/ # parallel BDD suite (behave) — shares helpers +docs/ # MkDocs site source + plans/ # the modernization roadmap lives here + decisions/ # ADRs (build backend, Python floor, etc.) +.github/workflows/ # ci.yml + release.yml + docs.yml +``` + +## Consumer guidance for agents using `rglob` + +If you are an agent consuming this package in another project, prefer the +stable machine surfaces: + +```bash +pip install rglob +rglob describe find +rglob schema grep +rglob schema --all +rglob capabilities --json +``` + +For Python integrations, import from `rglob.agent`, not private modules: + +```python +from pathlib import Path + +from rglob.agent import WalkOptions, search_all + +result = search_all(WalkOptions(patterns=["*.py"], base=Path("."))) +``` + +For MCP hosts: + +```bash +pip install "rglob[mcp]" +rglob mcp +``` + +Structured outputs are bounded with `--limit`, `--max-bytes`, and +`--max-file-size`; they report `truncated` and `truncated_reason` when a +limit is hit. Operational failures are returned as `ErrorInfo` records or a +stable error envelope. See `docs/agents/` for setup and safety details. + +## Mandatory conventions + +**All branches, PR titles, and commits MUST follow +[Conventional Commits](https://www.conventionalcommits.org/) v1.0.0.** + +- **Commit / PR title**: `[(scope)][!]: ` + - Types in use: `feat`, `fix`, `refactor`, `perf`, `docs`, `test`, + `build`, `ci`, `chore`, `style`, `revert`. + - Append `!` after the type/scope when the change is breaking + (`feat!: drop X`, `refactor(walker)!: ...`). + - Description starts lowercase, no trailing period, imperative mood + ("add", "drop", not "added", "drops"). + - Soft 72-char limit on the subject line. +- **Branch names**: matching prefixes — `feat/...`, `fix/...`, + `refactor/...`, `docs/...`, `chore/...`, `release/...`. Use kebab-case + after the slash (`feat/scandir-walker`, `release/2.0`, not + `feat/Scandir_Walker`). +- **Body**: motivate the *why*; reference issues with `Closes #N`; record + breaking changes under a `BREAKING CHANGE:` footer when the description + alone isn't enough. + +CI does not currently fail on a non-conventional title, but reviewers will +ask you to rename. The next-major changelog generator assumes the +convention. + +## Developer workflow + +```bash +make dev-setup # one-time: installs [dev,bdd,docs,gitignore,ext,bench] + # plus `pre-commit install` +make help # list all targets +make lint # ruff check + ruff format --check + mypy --strict +make test # pytest --cov-fail-under=100 + behave (both gating) +make bench # pytest-benchmark walker benchmarks; rg comparison skips if absent +make fmt # auto-format with ruff +make docs # live MkDocs preview on :8000 +make docs-build # static site build (strict mode) +make build # hatch build → dist/{sdist,wheel} +make publish # hatch build + twine check/upload to PyPI +make clean # remove caches and build outputs +``` + +Per-job CI uses `--cov-fail-under=95`; the **merged** Codecov report must +hit **100%**. Locally the gate is 100% on a single Linux run. See +[ADR-0006](docs/decisions/0006-aggregated-coverage.md). + +## Style gates + +- **Ruff** with `select = ["E","F","W","I","B","UP","SIM","RUF","PTH", + "PERF","D"]`, Google docstring convention, 100-char lines. Config in + `pyproject.toml`. +- **Mypy** `strict = true` on `src/rglob`. Ship `py.typed`. +- **`# pragma: no cover`** is reserved for genuinely unreachable lines + (`if __name__ == "__main__":` guards, cross-platform branches that no + matrix cell covers). Every pragma carries a one-line justification. + +## Where to look first + +- **Roadmap**: [`docs/plans/modernization-roadmap.md`](docs/plans/modernization-roadmap.md) + records the six-phase plan (plus the agent-platform sub-phases) that + delivered 2.0. Useful as historical context when changing a subsystem. +- **Decisions log**: [`docs/decisions/`](docs/decisions/) captures the + locked-in choices (build backend, Python floor, Path return, Typer + rationale, coverage strategy, behave-as-parallel, security model). +- **Architecture**: [`docs/architecture.md`](docs/architecture.md) has + Mermaid diagrams of the walker, CLI hierarchy, dupes pipeline, + `.gitignore` flow, and the 2.0 public API. + +## Working with the existing code + +- The walker in `src/rglob/rglob.py` is the hot path. Changes there should + preserve `os.scandir`'s cached `DirEntry.d_type` behaviour (don't add + unnecessary `stat()` calls). +- The CLI in `src/rglob/cli.py` uses Typer; new subcommands go below the + existing block. Shared filter `Annotated` aliases (`BaseOpt`, + `ExcludeOpt`, etc.) are defined at the top of the module — reuse them. +- Behave step bodies (`features/steps/steps.py`) should call the same + helpers `tests/conftest.py::TreeBuilder` exposes; the two suites must + agree. + +## Things to NOT do + +- Don't add `from __future__ import annotations` to `src/rglob/*.py` — + PEP 604 unions work natively on the 3.11 floor. +- Don't introduce `os.path` calls in new code; use `pathlib.Path`. +- Don't add new optional dependencies without a matching + `[project.optional-dependencies]` extra in `pyproject.toml`. +- Don't bump the trove `Development Status` classifier without coordinating + with the maintainer — it's currently `6 - Mature` post-2.0. +- Don't disable the 100% coverage gate. If a line is truly unreachable, + use `# pragma: no cover` with a one-line justification; if it's + reachable, write a test. +- Don't skip pre-commit / signed-commit hooks (`--no-verify`, + `--no-gpg-sign`) without explicit maintainer instruction. + +## Releases + +The 2.0 release is the only release point so far. Future releases follow +SemVer; tags are `vMAJOR.MINOR.PATCH` and trigger +`.github/workflows/release.yml`, which builds with hatch and publishes via +PyPI OIDC trusted publishing. Source `__version__` is bumped before the +tag is pushed. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..d620a13 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,190 @@ +# Changelog + +All notable changes to `rglob` will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [2.0.0] – 2026-05-18 + +The 2.0 release lands a top-to-bottom modernization across six phases. See +[`docs/plans/modernization-roadmap.md`](docs/plans/modernization-roadmap.md) +for the full plan and [`docs/migrating-to-2.0.md`](docs/migrating-to-2.0.md) +for the one-line `str ↔ Path` migration. + +### Changed — Breaking (Phase 6) +- **`rglob()` and `rglob_()` now return `list[Path]`** (was `list[str]` in + 1.x). Migration is one line: `[str(p) for p in rglob(...)]`. See + [migrating-to-2.0.md](docs/migrating-to-2.0.md) and ADR-0003 for context. +- Version bumped to `2.0.0`; `Development Status` classifier raised to + `6 - Mature`. + +### Added (Phase 1 — Packaging hygiene) +- PEP 621 metadata in `pyproject.toml` with `hatchling` build backend. +- Single-source `__version__` in `src/rglob/__init__.py` (now `2.0.0`). +- `src/` layout — package now lives at `src/rglob/`. +- `CHANGELOG.md`, `CONTRIBUTING.md`, `SECURITY.md`, `CODE_OF_CONDUCT.md`. +- Top-level ASCII banner in `README.md`; refreshed badges and sections. + +### Changed (Phase 1) +- Minimum Python version raised from 3.5 to **3.11** (3.10 reaches EOL October + 2026 — the 2.0 release ships on a forward-looking floor). +- `setup.py` deleted; all metadata now in `pyproject.toml`. +- Trove classifiers refreshed (3.11–3.14, `Topic :: System :: Filesystems`, + `Environment :: Console`, `Typing :: Typed`, `Development Status :: 5 - Production/Stable`). +- `.gitignore` expanded to cover modern Python dev artefacts (`dist/`, `.venv/`, + `.pytest_cache/`, `.mypy_cache/`, `.ruff_cache/`, coverage outputs, etc.). +- README rebuilt with an ASCII banner, badges row, refreshed sections, and the + legacy `**`/sorted-paths disclaimers removed (both behaviours flip in Phase 3). + +### Added (Phase 2 — Tooling overhaul) +- Ruff (lint + format) replaces pylint with a curated lint preset (`E`, `F`, + `W`, `I`, `B`, `UP`, `SIM`, `RUF`, `PTH`, `PERF`, `D`). +- mypy `strict = true` on `src/rglob` with a `py.typed` marker. +- Pytest test suite ported from the Behave scenarios; `tests/` is now the + primary suite. Behave runs in parallel (see ADR-0007). +- Aggregated 100% coverage gate (see ADR-0006): local enforces `--cov-fail-under=100`, + CI per-job uses 95% with the Codecov merged report at 100%. +- New GitHub workflows: `ci.yml` (matrix 3.11–3.14 × ubuntu/macos/windows), + `release.yml` (OIDC PyPI publishing), `docs.yml` (gh-pages deploy). +- MkDocs Material site with Mermaid support, mkdocstrings API page, ADR + collection in `docs/decisions/`. +- `Makefile` with `help`, `build`, `lint`, `test`, `fmt`, `docs`, `docs-build`, + `dev-setup`, and `clean` targets. +- `.pre-commit-config.yaml` (ruff, mypy, pyproject-fmt, hygiene hooks). +- `renovate.json` (weekly schedule, dependency dashboard). +- `[project.optional-dependencies]`: `dev`, `bdd`, `docs`, `gitignore`, `ext`, + `bench`. + +### Removed (Phase 2) +- `.coveragerc` (config moved to `[tool.coverage.*]` in `pyproject.toml`). +- `requirements-dev.txt` (replaced by optional extras). +- `.github/workflows/pylint.yml` (replaced by `ci.yml`). +- pylint sections in `pyproject.toml`. + +### Added (Phase 3 — Core API on `os.scandir`) +- New canonical API: `find(base, patterns, *, exclude=(), max_depth=None, + hidden=False, follow_symlinks=False, case_sensitive=None, sort=True, + on_error="warn") -> Iterator[Path]` and `find_all(...) -> list[Path]`. +- `**` recursive glob support (`find_all("src", "**/*.py")`). +- Symlink loop detection via per-call `os.path.realpath` memo. +- Deterministic sort-by-default ordering; opt-out via `sort=False`. +- OS-default case sensitivity (`case_sensitive=None`): case-sensitive on + Linux, case-insensitive on macOS/Windows. +- Error handling knob (`on_error="ignore"|"warn"|"raise"`) covers + `PermissionError`, `OSError`, `UnicodeDecodeError`. +- Hypothesis property tests: cardinality vs `os.walk`, `max_depth` + monotonicity, exclude commutativity, case-sensitive ⊆ case-insensitive. + +### Changed (Phase 3) +- Walker internals rewritten on `os.scandir` (drops a `stat()` per entry by + reading cached `DirEntry.d_type`). +- Legacy `rglob` / `rglob_` / `lcount` / `tsize` now thin wrappers over + `find()` — return types unchanged (`list[str]` / `int` / `float`). +- Dropped `from __future__ import annotations` from `src/rglob/rglob.py` + (PEP 604 unions work natively on the 3.11 floor). + +### Added (Phase 4 — CLI overhaul on Typer + Rich) +- CLI rewritten on Typer; `find` / `lcount` / `tsize` keep their legacy + flags (`--base`, `--no-empty`, `--no-comments`, `--unit`) byte-compatibly. +- New `find` filter flags: `-E/--exclude`, `-d/--max-depth`, `-H/--hidden`, + `-L/--follow`, `-s/-i / --case-sensitive/--case-insensitive`, + `--gitignore/--no-gitignore` (stub — full behaviour lands in Phase 5). +- New `find` output formats: `--json` (array), `--jsonl` (object per line), + `-0/--null` (for `xargs -0`), `--format TEMPLATE` (mini-template with + `path`, `rel`, `name`, `size`, `size_kb`, `size_mb` fields). +- Rich integration: coloured help, coloured warnings (e.g. shell + pre-expansion detection). `NO_COLOR` is respected automatically by Rich. +- Multiple positional patterns on `find` are OR'd + (`rglob find "*.py" "*.pyx"`). +- `rglob` now warns to stderr when called with multiple positional + arguments that look pre-expanded by the shell. +- Shell completion via Typer: + `rglob --install-completion {bash,zsh,fish,powershell}`. +- `docs/cli.md` now renders the auto-generated command tree via + `mkdocs-typer2`. +- Snapshot tests via `syrupy` (`tests/test_cli.py`) cover the find / + lcount / tsize golden outputs. + +### Added (Phase 5 — Fun features) +- Tier 2 filters wired into both `find()` and the `find` CLI command: + - `min_size`/`max_size` accepting `int | float | str` ("1K", "5MiB", etc.). + - `newer_than`/`older_than` accepting `datetime | timedelta | str` + ("7d", "2024-01-01", etc.). + - `kinds: set[Literal["f","d","l","x"]]` (file/dir/symlink/executable). +- `.gitignore` awareness via the optional `pathspec` extra + (`pip install rglob[gitignore]`). New `respect_gitignore: bool = False` + kwarg on `find()`; `--gitignore/--no-gitignore` flag on `rglob find`. +- New CLI subcommands: + - `rglob stats ` — summary table (count, size, extension breakdown). + - `rglob tree ` — Rich-rendered Unicode tree (default depth 3). + - `rglob top -n N` — top-N largest files. + - `rglob dupes ` — duplicate-file detection (size → 4-KiB hash + → full hash). Uses `xxhash` when installed (`pip install rglob[ext]`), + stdlib BLAKE2b otherwise. +- New `find` CLI flags: `-t/--type`, `--min-size`, `--max-size`, + `--newer-than`, `--older-than`. +- New internal modules: `src/rglob/_filters.py` (size/time/kind parsers and + predicates, `.gitignore` matcher) and `src/rglob/_dupes.py` + (duplicate-finding pipeline). + +### Added (Agent platform Phase 0 — Contract and schemas) +- ADR-0009 documents the SemVer-locked agent API contract for `rglob.agent`, + structured CLI JSON, JSON Schemas, and MCP tools. +- ADR-0010 documents the read-only safety model for agent and MCP use: + base containment, symlink behavior, content disclosure, binary files, + unreadable paths, output limits, and cooperative timeouts. +- New frozen, slotted contract models in `src/rglob/agent/_models.py`: + `FileMatch`, `LineMatch`, `Stats`, `Duplicate`, `ErrorInfo`, `ErrorCode`, + concrete search result envelopes, capability reports, and option models. +- Runtime JSON Schema Draft 2020-12 generation from + `src/rglob/agent/_models.py`, exposed through `rglob.agent.schema_for`, + `rglob.agent.all_schemas`, `rglob schema `, and + `rglob schema --all`. +- New machine-readable CLI endpoints: + `rglob describe `, `rglob schema `, + `rglob capabilities --json`, and `rglob agent-version`. +- `rglob find --json` now emits a `FileSearchResult` object instead of a + string array. `rglob find --jsonl` emits the same result shape as one + compact JSON object line. +- New shared golden fixture tree at `tests/fixtures/agent-tree/` for agent + contract, binary, hidden-file, gitignore, unreadable-path, and duplicate + scenarios. + +### Added (Agent platform Phase 1 — Scope expansion) +- New `rglob grep` command and `src/rglob/_grep.py` content matcher returning + `LineSearchResult` records. +- New `rglob count` command and structured `Stats` output for files, lines, + and bytes. +- `find()` and `rglob find` gained `--perm`, `--uid`, `--gid`, and + `--newer-than-file` predicates. +- `rglob grep` supports fixed strings, ignore-case, context, max-count, + word, invert, encoding, binary-as-text, limits, and JSON / JSONL output. + +### Added (Agent platform Phase 2 — Python API) +- New stable `rglob.agent` namespace exporting the agent dataclasses, + `__agent_api_version__`, `search`, `search_all`, `grep`, `grep_all`, + `count`, and `find_duplicates`. +- Agent API operational errors are represented as `ErrorInfo` records instead + of raising for bad predicates, unreadable files, binary skips, and regex + failures. + +### Added (Agent platform Phase 3 — MCP) +- New optional `mcp` extra: `pip install "rglob[mcp]"`. +- New `rglob mcp` command and `src/rglob/agent/mcp.py` stdio MCP server. +- MCP tools: `find_files`, `grep_content`, `count_lines`, + `find_duplicate_files`, and `describe_subcommand`. + +### Added (Agent platform Phase 4 — Documentation) +- New `docs/agents/` pages for Python API usage, MCP setup, CLI recipes, + stability, and safety. +- New `docs/examples/agent-recipes.md` with copy-paste agent workflows. +- `AGENTS.md` now includes consumer guidance for agents using `rglob`, in + addition to contributor instructions for agents editing this repository. + +## Historical releases (pre-2.0) + +Earlier releases (`1.2`–`1.7`, plus a `v1.8` tag with no matching source bump) +were published to PyPI without a structured changelog. See the +[PyPI release history](https://pypi.org/project/rglob/#history) for the rough +timeline. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..3cc2cbc --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,37 @@ +# Code of Conduct + +This project adopts the **Contributor Covenant**, version 2.1, as its code of +conduct. + +The full text is published canonically at: + + + +A plain-text mirror is also kept at: + + + +By participating in this project — opening issues, submitting pull requests, +discussing in reviews — you agree to abide by its terms. + +## Reporting + +If you believe someone has violated this code of conduct, please open a +[private security advisory](https://github.com/chris-piekarski/python-rglob/security/advisories/new) +or email the maintainer directly at the address listed in `pyproject.toml`. + +Reports are handled confidentially. You can expect an acknowledgement within a +week. + +## Enforcement + +`rglob` is a small hobby project and does not have a formal enforcement +committee. The maintainer (Chris Piekarski) will review reports in good faith +and apply the Contributor Covenant enforcement ladder as appropriate +(warning → temporary ban → permanent ban) when needed. + +## Attribution + +This Code of Conduct is adapted from the +[Contributor Covenant](https://www.contributor-covenant.org), version 2.1, +available at the URL above. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..a52aab5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,67 @@ +# Contributing to `rglob` + +Thanks for your interest! `rglob` is a small, fun, labor-of-love package — contributions in that spirit are very welcome. + +## Quick start + +```bash +git clone https://github.com/chris-piekarski/python-rglob.git +cd python-rglob +python -m venv .venv && source .venv/bin/activate +make dev-setup # installs .[dev,bdd,docs] and pre-commit hooks +make test # pytest + behave, gated at 100% coverage +make lint # ruff + mypy --strict +``` + +## Standards we enforce + +- **Lint**: `ruff check` and `ruff format --check` must pass. +- **Types**: `mypy --strict src/rglob` must pass. +- **Tests**: `pytest --cov --cov-fail-under=100` and `behave` must pass. +- **Coverage**: 100% line coverage on `src/rglob/`. Use `# pragma: no cover` only + for genuinely un-coverable lines and document the reason inline. +- **Conventional commits** are nice but not required. + +CI runs the full matrix (Python 3.11–3.14 × Ubuntu/macOS/Windows). Branch +protection on `master` blocks merges without all checks green. + +### Cross-platform test guidelines + +The Ubuntu, macOS, and Windows jobs share the same `tests/` tree, so new tests +must be portable or explicitly opt out: + +- **POSIX mode bits** (`chmod`, `os.getuid`, exec-kind detection) — wrap the + test in `@posix_only` (defined in `tests/test_filters.py`) or + `@pytest.mark.skipif(os.name != "posix", reason=…)`. NTFS does not honour + `Path.chmod()` for executable bits. +- **Filesystem case sensitivity** — APFS (macOS default) and NTFS treat + `README.MD` and `readme.md` as one inode. Tests that need both casings as + distinct files should request the `case_sensitive_fs` fixture (in + `tests/conftest.py`) and `pytest.skip()` when it returns `False`. +- **Symlinks** — Windows requires Developer Mode or admin to create symlinks. + Wrap `Path.symlink_to()` in `try/except (OSError, NotImplementedError)` and + `pytest.skip()` on failure. +- **`bash` subprocesses** — Windows GA runners ship a WSL stub at + `C:\Windows\System32\bash.exe` that exits non-zero when no distro is + installed. Probe with `bash -c true` and check the return code (see + `tests/test_examples.py::_has_bash`). + +The `--cov-fail-under=95` gate only fires on Ubuntu because macOS/Windows +legitimately skip POSIX-only branches. + +## Filing a bug or feature request + +Open an issue at . +For security issues, please follow `SECURITY.md` instead. + +## Pull request flow + +1. Fork & branch off `master`. +2. Make your change; add tests; update `CHANGELOG.md` under `[Unreleased]`. +3. Run `make lint test` locally. +4. Open a PR. CodeRabbit / CI will leave comments — address them. +5. A maintainer will merge once green. + +## Code of Conduct + +By participating, you agree to abide by the [Contributor Covenant](CODE_OF_CONDUCT.md). diff --git a/Makefile b/Makefile index 5727e45..a760952 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,58 @@ -.PHONY: lint +.DEFAULT_GOAL := help -lint: - @echo "Running pylint on package, setup, and tests..." - pylint rglob setup.py features +# Auto-prefer the project's virtualenv when one exists. Falls back to +# `python3` on PATH so CI (which runs without `.venv/`) still works. +PYTHON ?= $(if $(wildcard .venv/bin/python),$(CURDIR)/.venv/bin/python,python3) +# Direct script invocation for mkdocs — `python -m mkdocs` puts the CWD on +# sys.path, which confuses mkdocstrings into trying to import the +# `mkdocs-typer2` plugin as a Python object and the build fails. +MKDOCS ?= $(if $(wildcard .venv/bin/mkdocs),$(CURDIR)/.venv/bin/mkdocs,mkdocs) +PACKAGE := rglob -.PHONY: dev-setup +.PHONY: help repo-stats build publish lint test bench fmt docs docs-build dev-setup clean -dev-setup: - @echo "Installing dev dependencies (pylint, behave)..." - pip install -r requirements-dev.txt +help: ## Show this help menu + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " \033[36m%-14s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) + +repo-stats: ## Update OVERVIEW.md with repo LOC stats and Mermaid chart + $(PYTHON) -m scripts.generate_repo_stats + +build: ## Build sdist + wheel into dist/ (uses hatch) + $(PYTHON) -m hatch build + +publish: build ## Check and upload dist/* to PyPI via twine + $(PYTHON) -m twine check dist/* + $(PYTHON) -m twine upload dist/* + +lint: ## Run ruff + mypy --strict (gating) + $(PYTHON) -m ruff check . + $(PYTHON) -m ruff format --check . + $(PYTHON) -m mypy --strict src/$(PACKAGE) + +test: ## Run pytest with coverage + behave (gating; local 100% coverage) + $(PYTHON) -m pytest --cov=$(PACKAGE) --cov-branch --cov-report=term-missing --cov-fail-under=100 + $(PYTHON) -m behave + +bench: ## Run pytest-benchmark performance checks + @$(PYTHON) -c "import pytest_benchmark" >/dev/null 2>&1 || { echo "Install benchmark dependencies with: make dev-setup"; exit 1; } + $(PYTHON) -m pytest bench/ --benchmark-only + +fmt: ## Auto-format with ruff (modifies files in place) + $(PYTHON) -m ruff format . + $(PYTHON) -m ruff check --fix . + +docs: ## Serve MkDocs preview on http://localhost:8000 + $(MKDOCS) serve + +docs-build: ## Build static docs site to ./site + $(MKDOCS) build --strict + +dev-setup: ## Install dev extras + pre-commit hooks + $(PYTHON) -m pip install -e ".[dev,bdd,docs,gitignore,ext,bench]" + $(PYTHON) -m pre_commit install + +clean: ## Remove caches, build artefacts, coverage outputs, docs site + rm -rf dist build site \ + .pytest_cache .mypy_cache .ruff_cache htmlcov \ + .coverage .coverage.* coverage.xml + find . -type d -name __pycache__ -prune -exec rm -rf {} + 2>/dev/null || true diff --git a/OVERVIEW.md b/OVERVIEW.md new file mode 100644 index 0000000..933fe9e --- /dev/null +++ b/OVERVIEW.md @@ -0,0 +1,52 @@ +# rglob — Repository Overview + +Auto-generated repository stats live below; everything outside +the `REPO-STATS` markers is hand-written. + + + +## Repository Stats + +- Code LOC (approx): 9,919 +- Total lines (tracked files): 12,482 + +```mermaid +pie title Code LOC by Area + "src" : 3212 + "tests" : 2846 + "docs" : 2038 + "other" : 1181 + "features" : 282 + "scripts" : 196 + "bench" : 164 +``` + +### LOC by Area +| Area | Code LOC | Total Lines | +|------|----------|-------------| +| src | 3,212 | 3,864 | +| tests | 2,846 | 3,815 | +| docs | 2,038 | 2,526 | +| other | 1,181 | 1,410 | +| features | 282 | 368 | +| scripts | 196 | 249 | +| bench | 164 | 250 | + +### Code LOC by Module (Python only) +```mermaid +pie title Code LOC by Module (src/rglob, Python only) + "root" : 2047 + "agent" : 1165 +``` + +### Top Python Modules (src/rglob) +| Module (src/rglob, .py only) | Code LOC | Total Lines | +|------------------------------|----------|-------------| +| root | 2,047 | 2,506 | +| agent | 1,165 | 1,358 | + +_Note: LOC approximates non-blank, non-comment lines. Module breakdown counts only Python files._ + + + + diff --git a/README.md b/README.md index 9678db5..73d48c0 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,47 @@ -

rglob

+ + +```text + __ __ + _________ _______/ /___ / /_ + / ___/ __ `/ ___/ / __ \/ __ \ + / / / /_/ / /__/ / /_/ / /_/ / +/_/ \__, /\___/_/\____/_.___/ + /____/ +``` + +

Lightweight recursive search for Python, CLIs, and coding agents.

+ +

+ + PyPI + Python versions + + CI + Coverage + mypy: strict + pre-commit + + License + Conventional Commits + Ruff + + GitHub stars + PyPI - Downloads + + Docs +

+ +--- -Lightweight recursive glob helpers for Python. Find files recursively, count lines, and sum sizes with simple functions. +## Why use this? -Python 3 users should install the latest release. For Python 2, the final supported version is 1.4 (links below). +Modern Python has `pathlib.Path.rglob`, and external tools like `fd` and +`ripgrep` are blazingly fast. `rglob` is smaller and more embeddable: filename +globbing, content grep, count/stat helpers, stable JSON schemas, a typed +`rglob.agent` namespace, and an optional MCP server. -— +It is designed to be a default recursive-search dependency for coding agents +that need predictable outputs, bounded result shapes, and read-only behavior. ## Installation @@ -12,16 +49,25 @@ Python 3 users should install the latest release. For Python 2, the final suppor pip install rglob ``` -## Quick Start +## Quick Start (Python API) ```python import rglob -# List all Python files under a base directory -files = rglob.rglob("/path/to/project", "*.py") +# Modern API — yields pathlib.Path, with filter kwargs +for p in rglob.find("/path/to/project", "*.py", exclude=".venv"): + print(p) + +# Eager variant +paths = rglob.find_all("/repo", "**/*.py", hidden=False, max_depth=4) + +# OS-aware case sensitivity (`None` = OS default — case-sensitive on Linux, +# case-insensitive on macOS/Windows; pass True/False to force) +paths = rglob.find_all(".", "*.PY", case_sensitive=False) -# Same, starting from the current working directory -files_cwd = rglob.rglob_("*.py") +# Legacy API still works (now returns list[Path] at 2.0 — see migration guide) +files = rglob.rglob("/path/to/project", "*.py") # → list[Path] +files_cwd = rglob.rglob_("*.py") # → list[Path] # Count non-empty, non-comment lines across matching files non_empty_non_comment = rglob.lcount( @@ -34,85 +80,145 @@ non_empty_non_comment = rglob.lcount( total_mb = rglob.tsize("/path/to/photos", "*.jpg", rglob.megabytes) ``` -Note: When invoking from a shell, quote or escape glob patterns so your shell doesn’t expand them before Python runs, e.g. `"*.py"`. +Agent integrations should import from the stable `rglob.agent` namespace: -## API - -- `rglob.rglob(base: str, pattern: str) -> list[str]`: Recursively returns a list of paths matching `pattern` under `base`. -- `rglob.rglob_(pattern: str) -> list[str]`: Same as `rglob`, using the current working directory as `base`. -- `rglob.lcount(base: str, pattern: str, func: Callable[[str], bool] = lambda _: True) -> int`: Counts lines across all matching files, applying `func` as a per-line predicate. -- `rglob.tsize(base: str, pattern: str, func: Callable[[float], float] = rglob.megabytes) -> float`: Sums sizes (in bytes) of matching files, then converts using `func`. -- Unit helpers: `rglob.kilobytes`, `rglob.megabytes`, `rglob.gigabytes`, `rglob.terabytes`. +```python +from pathlib import Path -## Command-Line Interface +from rglob.agent import GrepOptions, WalkOptions, grep_all, search_all -`rglob` can also be used as a command-line tool. +files = search_all(WalkOptions(patterns=["*.py"], base=Path("src"))) +todos = grep_all(GrepOptions(pattern="TODO", paths=["*.py"], base=Path("src"))) +``` -Important: quote your patterns +Paths are sorted by default for deterministic output. Pass `sort=False` for +raw `scandir` order. Recursive `**` globs work +(`rglob.find_all("src", "**/*.py")`); symlink loops are detected and +terminated automatically. -- Always quote or escape glob patterns so your shell does not expand them before Python runs. -- Wrong: `python3 -m rglob.cli find *.py` (the shell expands `*.py` first) -- Right: `python3 -m rglob.cli find "*.py"` -- Installed entry point works the same: `rglob find "*.py"` +> Upgrading from 1.x? `rglob()` now returns `list[Path]` instead of +> `list[str]`. See [migrating to 2.0](docs/migrating-to-2.0.md) for the +> one-line migration. -### Find files +## Quick Start (CLI) ```bash +# Find files rglob find "*.py" -``` -### Count lines +# Multiple patterns are OR'd +rglob find "*.py" "*.pyx" -```bash +# Filter flags +rglob find "*.py" --base ./src --exclude .venv -d 3 --hidden + +# Output formats +rglob find "*.py" --json | jq '.results[] | .path' +rglob find "*.py" --jsonl +rglob find "*.py" -0 | xargs -0 wc -l # NUL-separated for xargs + +# Mini-template formatter +rglob find "*.py" --format "{name}: {size_mb:.2f} MiB" + +# Count lines, skipping empties and comment lines rglob lcount "*.py" --no-empty --no-comments -``` -### Get total size +# Grep content and count structured stats +rglob grep TODO "*.py" --context 2 --json +rglob count "*.py" --no-empty --no-comments --json -```bash +# Sum total size in MB rglob tsize "*.py" --unit mb + +# Machine discovery for agents +rglob describe find +rglob schema grep +rglob schema --all +rglob capabilities --json +rglob agent-version # locked SemVer of the agent contract (see ADR-0009) + +# MCP server (stdio). Exposes `find_files`, `grep_content`, `count_lines`, +# `find_duplicate_files`, and `describe_subcommand` with read-only, +# bounded defaults. Full setup in docs/agents/mcp-setup.md. +pip install "rglob[mcp]" +rglob mcp + +# Shell completion (one-time setup) +rglob --install-completion bash # or zsh / fish / powershell ``` -### Tips +> **Quote your patterns!** Otherwise your shell pre-expands them before Python +> runs. Use `rglob find "*.py"`, not `rglob find *.py`. If `rglob find` receives +> multiple unquoted positional patterns it will warn you on stderr. -- Paths returned are not guaranteed to be sorted; call `sorted(...)` if ordering matters. -- Patterns use Python’s `glob` syntax (e.g., `"*.py"`, `"**/*.py"` is not supported here; recursion is handled by `rglob`). -- Line counting opens files in text mode; ensure your files are decodable with the system default encoding. +## Fun features -## Compatibility +```bash +# Summary table: file count, total size, extension breakdown +rglob stats "*.py" --base ./src -- Python 3: Use the latest release (e.g., 1.7+ on PyPI). -- Python 2: Final supported release is 1.4. +# Unicode tree of matches (depth 3 by default) +rglob tree "*.py" --base ./src -Links: +# Top 10 largest files +rglob top "*" --base ~/Downloads -- PyPI (Py3): https://pypi.org/project/rglob/1.7/ -- PyPI (Py2): https://pypi.org/project/rglob/1.4/ +# Find duplicate files (size → 4-KiB hash → full hash) +rglob dupes "*" --base ~/Downloads --min-size 1M -## Development +# Respect .gitignore (requires `pip install rglob[gitignore]`) +rglob find "*" --gitignore -```bash -git clone https://github.com/chris-piekarski/python-rglob.git -cd python-rglob -python -m venv .venv && source .venv/bin/activate -pip install -e . +# Filter by kind / size / mtime +rglob find "*" -t f --min-size 1M --newer-than 7d ``` -### Running examples/tests +The duplicate detection uses `xxhash.xxh3_64` when the optional `[ext]` +extra is installed; it falls back to stdlib BLAKE2b otherwise — both are +fast enough that the difference rarely matters in 2026. -This repo includes BDD tests using `behave`. +## Compatibility -```bash -pip install behave -behave -``` +| Python | Status | +| --- | --- | +| 3.11+ | Supported (`rglob` 2.0+) | +| 3.10 | Pin `rglob<2` — dropped at 2.0 (Python 3.10 EOL is October 2026) | +| 3.6–3.9 | Not supported | +| 2.7 | Final supported release is `1.4` (PyPI history: ) | + +## Documentation + +Full docs (API reference, CLI reference, architecture diagrams, ADRs) live in +[`docs/`](docs/) and are published as a MkDocs Material site at +. + +- [Agent integration](docs/agents/index.md) — CLI JSON, Python API, MCP, safety, + and stability guidance for coding agents. +- [Modernization roadmap](docs/plans/modernization-roadmap.md) — the six-phase + plan that delivered 2.0. +- [Migrating to 2.0](docs/migrating-to-2.0.md) — the `list[str]` → `list[Path]` + return-type flip. +- [Architecture](docs/architecture.md) — package layout, walker call-graph, + CLI command hierarchy, `dupes` pipeline, and the 2.0 public-API class + diagram. +- [Decisions](docs/decisions/) — ADRs for the locked-in design choices. -Note: The core library supports Python 3 (see `setup.py`). +## Development -## Why use this? +```bash +git clone https://github.com/chris-piekarski/python-rglob.git +cd python-rglob +python -m venv .venv && source .venv/bin/activate +make dev-setup # installs [dev,bdd,docs,gitignore] + pre-commit hooks +make test # pytest + behave, gated at 100% local coverage +make lint # ruff + mypy --strict +make docs # live MkDocs preview at :8000 +``` -`rglob` predates `glob.glob(..., recursive=True)` and offers a tiny, easy-to-read implementation with convenient helpers for line counting and size aggregation. It’s useful for quick scripts and small utilities where pulling in heavier tools is unnecessary. +The 2.0 release replaced `pylint` with Ruff as the primary linter and added +`mypy --strict`. `make lint` runs both; `pylint src/rglob features` still +works if you want a second opinion. ## License -Apache 2.0 — see `LICENSE`. +Apache 2.0 — see [`LICENSE`](LICENSE). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..0859524 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,18 @@ +# Security Policy + +## Reporting a vulnerability + +`rglob` is a small filesystem-walking helper and not part of any critical +security boundary, but if you spot something that could harm users (e.g. a +path-traversal flaw, symlink-escape, or arbitrary file disclosure), please +report it privately rather than via a public issue. + +Open a [private security advisory on GitHub](https://github.com/chris-piekarski/python-rglob/security/advisories/new). + +You can expect an acknowledgement within a week. Fix timing depends on +severity and complexity. + +## Supported versions + +Only the latest minor release on PyPI is supported. Older versions may +contain known issues that are fixed only in the current line. diff --git a/bench/__init__.py b/bench/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bench/test_walk.py b/bench/test_walk.py new file mode 100644 index 0000000..25b3c37 --- /dev/null +++ b/bench/test_walk.py @@ -0,0 +1,250 @@ +"""Walker and grep performance benchmarks. + +Measures the in-process Python walker against the same task done with +``ripgrep`` for an external reference. Trees are built once per test +via ``tmp_path`` and the actual operation is wrapped in +``benchmark()`` so ``pytest-benchmark`` reports median timings and +operations-per-second. + +Run via: + + .venv/bin/python -m pytest bench/ --benchmark-only --benchmark-columns=median,ops,rounds +""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +import pytest + +from rglob import find_all +from rglob.agent import GrepOptions, grep_all + +# ─── Tree builders ──────────────────────────────────────────────────────────── + + +def _build_tree( + root: Path, + *, + dirs: int = 20, + files_per_dir: int = 50, + file_size: int = 64, +) -> int: + """Build a synthetic tree and return the total file count.""" + body = ("print('x')\n" * max(1, file_size // 16))[:file_size] or "x\n" + for dir_index in range(dirs): + directory = root / f"pkg_{dir_index:03d}" + directory.mkdir() + for file_index in range(files_per_dir): + suffix = ".py" if file_index % 2 == 0 else ".txt" + (directory / f"module_{file_index:04d}{suffix}").write_text(body) + return dirs * files_per_dir + + +def _seed_todos(root: Path, *, every: int = 5) -> int: + """Replace every Nth `.py` file's content with a TODO line; return count.""" + count = 0 + for index, path in enumerate(sorted(root.rglob("*.py"))): + if index % every == 0: + path.write_text("alpha\nTODO: kept for grep benchmark\nbeta\n") + count += 1 + return count + + +# ─── Walker benchmarks ─────────────────────────────────────────────────────── + + +def test_find_unsorted_python_files(benchmark, tmp_path): + """`find_all(..., sort=False)` over a 1k-file tree.""" + total = _build_tree(tmp_path) + + def run() -> list: + return find_all(tmp_path, "*.py", sort=False) + + result = benchmark(run) + assert len(result) == total // 2 + + +def test_find_sorted_python_files(benchmark, tmp_path): + """`find_all(..., sort=True)` is the default ordering mode.""" + total = _build_tree(tmp_path) + + def run() -> list: + return find_all(tmp_path, "*.py", sort=True) + + result = benchmark(run) + assert len(result) == total // 2 + + +def test_find_all_files_star(benchmark, tmp_path): + """`find_all(..., "*")` should hit the `*`-pattern fast path.""" + total = _build_tree(tmp_path) + + def run() -> list: + return find_all(tmp_path, "*", sort=False) + + result = benchmark(run) + # `*` matches everything: files (`total`) plus the `dirs` directories. + assert len(result) >= total + + +def test_find_only_files_kind(benchmark, tmp_path): + """`kinds={'f'}` should hit the kind fast-path (no stat() per entry).""" + total = _build_tree(tmp_path) + + def run() -> list: + return find_all(tmp_path, "*", sort=False, kinds={"f"}) + + result = benchmark(run) + assert len(result) == total + + +def test_find_only_dirs_kind(benchmark, tmp_path): + """`kinds={'d'}` returns only directories.""" + _build_tree(tmp_path, dirs=20, files_per_dir=50) + + def run() -> list: + return find_all(tmp_path, "*", sort=False, kinds={"d"}) + + result = benchmark(run) + assert len(result) == 20 + + +def test_find_large_files(benchmark, tmp_path): + """Walker cost should be dominated by entry count, not file content size.""" + total = _build_tree(tmp_path, dirs=10, files_per_dir=50, file_size=4096) + + def run() -> list: + return find_all(tmp_path, "*.py", sort=False) + + result = benchmark(run) + assert len(result) == total // 2 + + +# ─── Grep benchmarks ───────────────────────────────────────────────────────── + + +def test_grep_todo_python(benchmark, tmp_path): + """`grep_all(...)` over a tree with planted TODOs (no early termination).""" + _build_tree(tmp_path) + seeded = _seed_todos(tmp_path) + + opts = GrepOptions(pattern="TODO", paths=["*.py"], base=tmp_path) + + def run(): + return grep_all(opts) + + result = benchmark(run) + assert result.total_files_searched == 500 # only `.py` files visited + assert len(result.results) == seeded + + +def test_grep_todo_python_with_limit(benchmark, tmp_path): + """`grep_all(..., limit=10)` should short-circuit after 10 matches.""" + _build_tree(tmp_path) + _seed_todos(tmp_path) + + opts = GrepOptions(pattern="TODO", paths=["*.py"], base=tmp_path, limit=10) + + def run(): + return grep_all(opts) + + result = benchmark(run) + assert result.truncated is True + assert result.truncated_reason == "limit" + assert len(result.results) == 10 + + +def test_grep_todo_large_files_with_limit(benchmark, tmp_path): + """Streaming early-break wins biggest on large files with a small limit.""" + # Larger files (~100 KB each) so the streaming branch fires. + _build_tree(tmp_path, dirs=5, files_per_dir=10, file_size=100_000) + _seed_todos(tmp_path, every=2) + + opts = GrepOptions(pattern="TODO", paths=["*.py"], base=tmp_path, limit=5) + + def run(): + return grep_all(opts) + + result = benchmark(run) + assert len(result.results) == 5 + + +def test_grep_files_with_matches(benchmark, tmp_path): + """`files_with_matches=True` stops reading each file at the first match.""" + _build_tree(tmp_path, dirs=10, files_per_dir=20, file_size=10_000) + _seed_todos(tmp_path, every=3) # ~33 matching .py files + + opts = GrepOptions(pattern="TODO", paths=["*.py"], base=tmp_path, files_with_matches=True) + + def run(): + return grep_all(opts) + + result = benchmark(run) + # Stub LineMatch per matching file (line_number=0, content=""). + assert all(m.line_number == 0 for m in result.results) + assert len(result.results) > 0 + + +def test_grep_count_only(benchmark, tmp_path): + """`count_only=True` scans every file but builds one record per match group.""" + _build_tree(tmp_path, dirs=10, files_per_dir=20, file_size=10_000) + _seed_todos(tmp_path, every=3) + + opts = GrepOptions(pattern="TODO", paths=["*.py"], base=tmp_path, count_only=True) + + def run(): + return grep_all(opts) + + result = benchmark(run) + # One stub LineMatch per matching file with line_number = match count. + assert all(m.line_number > 0 for m in result.results) + + +# ─── Ripgrep parity benchmarks ─────────────────────────────────────────────── + + +@pytest.fixture(scope="module") +def _rg() -> str: + """Locate ripgrep; skip the comparison benchmarks when it's missing.""" + binary = shutil.which("rg") + if binary is None: + pytest.skip("ripgrep is not installed") + return binary + + +def test_rg_files_python_glob(benchmark, tmp_path, _rg): + """Reference: `rg --files -g '*.py'` over the same tree.""" + total = _build_tree(tmp_path) + + def run(): + return subprocess.run( + [_rg, "--files", "-g", "*.py", str(tmp_path)], + check=True, + capture_output=True, + text=True, + ) + + result = benchmark(run) + assert len(result.stdout.splitlines()) == total // 2 + + +def test_rg_grep_todo(benchmark, tmp_path, _rg): + """Reference: `rg --json TODO` against the same TODO-seeded tree.""" + _build_tree(tmp_path) + seeded = _seed_todos(tmp_path) + + def run(): + return subprocess.run( + [_rg, "--json", "-g", "*.py", "TODO", str(tmp_path)], + check=True, + capture_output=True, + text=True, + ) + + result = benchmark(run) + # Each rg JSON match record has `"type":"match"`. + match_lines = [line for line in result.stdout.splitlines() if '"type":"match"' in line] + assert len(match_lines) == seeded diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..c3fd1b4 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,24 @@ +coverage: + status: + project: + default: + target: 100% + threshold: 0% + informational: false + patch: + default: + target: 100% + threshold: 0% +comment: + layout: "header, diff, flags, components" + require_changes: true + +flag_management: + default_rules: + carryforward: true + +ignore: + - "tests/**" + - "features/**" + - "bench/**" + - "docs/**" diff --git a/docs/agents/cli-recipes.md b/docs/agents/cli-recipes.md new file mode 100644 index 0000000..b4eae0d --- /dev/null +++ b/docs/agents/cli-recipes.md @@ -0,0 +1,27 @@ +# CLI Recipes + +Find Python files as structured JSON: + +```bash +rglob find "*.py" --base src --json +``` + +Grep TODOs with context: + +```bash +rglob grep TODO "*.py" --base src --context 2 --json +``` + +Count non-empty, non-comment Python lines: + +```bash +rglob count "*.py" --base src --no-empty --no-comments --json +``` + +Discover command schemas: + +```bash +rglob schema find +rglob schema --all +rglob describe grep +``` diff --git a/docs/agents/index.md b/docs/agents/index.md new file mode 100644 index 0000000..3a666f1 --- /dev/null +++ b/docs/agents/index.md @@ -0,0 +1,23 @@ +# Agent Integration + +`rglob` 2.0 exposes three machine-friendly surfaces for coding agents: + +- bounded JSON / JSONL CLI output +- the typed `rglob.agent` Python API +- an optional stdio MCP server via `rglob[mcp]` + +The stable agent contract is versioned separately from the package: + +```bash +rglob agent-version +``` + +Use `rglob describe `, `rglob schema `, and +`rglob schema --all` for machine-readable discovery instead of scraping +`--help`. + +## Stability + +The SemVer-locked surface is documented in +[ADR-0009](../decisions/0009-agent-api-contract.md). The read-only safety +model is documented in [ADR-0010](../decisions/0010-agent-safety-model.md). diff --git a/docs/agents/mcp-setup.md b/docs/agents/mcp-setup.md new file mode 100644 index 0000000..b718f2c --- /dev/null +++ b/docs/agents/mcp-setup.md @@ -0,0 +1,32 @@ +# MCP Setup + +Install the optional extra: + +```bash +pip install "rglob[mcp]" +``` + +Then configure an MCP host to run: + +```bash +rglob mcp +``` + +The stdio server exposes these tools, all read-only with bounded +defaults (`limit=5000`, `follow_symlinks=False`, `strict_base=True`): + +- `find_files(pattern, base, ...walk filters)` +- `grep_content(pattern, paths, base, ...grep options, ...walk filters)` +- `count_lines(pattern, base, no_empty, no_comments, encoding, ...walk filters)` +- `find_duplicate_files(pattern, base, ...walk filters)` +- `describe_subcommand(name)` + +Each parameter is declared explicitly on the tool signature — FastMCP +introspects the signature to publish a JSON Schema, so agents discover +every field by name (no opaque `**kwargs`). The shared *walk filters* +are `exclude`, `max_depth`, `hidden`, `follow_symlinks`, +`case_sensitive`, `respect_gitignore`, `limit`, `max_bytes`, +`max_file_size`. Tool results use the same JSON-safe shapes as +`rglob.agent` and the structured CLI output; run `rglob schema --all` +for the matching agent-API contract. + diff --git a/docs/agents/python-api.md b/docs/agents/python-api.md new file mode 100644 index 0000000..cbec1a2 --- /dev/null +++ b/docs/agents/python-api.md @@ -0,0 +1,24 @@ +# Python API + +Use `rglob.agent` when an integration needs stable dataclasses and +non-raising operational behavior. + +```python +from pathlib import Path + +from rglob.agent import GrepOptions, WalkOptions, grep_all, schema_for, search_all + +files = search_all(WalkOptions(patterns=["*.py"], base=Path("src"))) +for match in files.results: + print(match.relative_path) + +todos = grep_all(GrepOptions(pattern="TODO", paths=["*.py"], base=Path("src"))) +for match in todos.results: + print(match.path, match.line_number, match.content) + +walk_schema = schema_for("walk_options") +``` + +Operational errors such as unreadable files, bad predicates, binary grep +skips, and regex failures are returned as `ErrorInfo` records on the result +object. Programmer errors can still raise. diff --git a/docs/agents/safety.md b/docs/agents/safety.md new file mode 100644 index 0000000..937d16e --- /dev/null +++ b/docs/agents/safety.md @@ -0,0 +1,17 @@ +# Safety + +The agent surfaces are read-only. They never edit, execute, chmod, or +delete files. + +Agent and MCP defaults favor bounded output and path containment: + +- `strict_base=True` +- `follow_symlinks=False` +- `include_errors=True` +- result limits are available on structured outputs +- binary grep is skipped unless `--text` / `text=True` is requested + +Structured results report truncation with `truncated` and +`truncated_reason`. Operational failures are returned as `ErrorInfo` +records or as the stable error envelope for command-level failures. + diff --git a/docs/agents/stability.md b/docs/agents/stability.md new file mode 100644 index 0000000..697d2d7 --- /dev/null +++ b/docs/agents/stability.md @@ -0,0 +1,15 @@ +# Stability + +The agent API version is exposed as `rglob.agent.__agent_api_version__` and +through `rglob agent-version`. + +Locked surfaces: + +- `rglob.agent` dataclasses and function signatures +- runtime-generated JSON Schemas +- structured CLI JSON / JSONL shapes +- MCP tool names and result shapes +- error codes and truncation metadata + +Adding fields or flags requires a minor agent API bump. Removing or +renaming locked fields requires a major bump. diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..7cfa458 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,67 @@ +# API Reference + +Auto-generated from docstrings via [mkdocstrings]. + +## Modern API + +::: rglob.rglob.find + options: + show_root_heading: true + show_source: false + +::: rglob.rglob.find_all + options: + show_root_heading: true + show_source: false + +## Legacy API (still supported) + +::: rglob.rglob.rglob + options: + show_root_heading: true + show_source: false + +::: rglob.rglob.rglob_ + options: + show_root_heading: true + show_source: false + +::: rglob.rglob.lcount + options: + show_root_heading: true + show_source: false + +::: rglob.rglob.tsize + options: + show_root_heading: true + show_source: false + +## Unit helpers + +::: rglob.rglob.kilobytes + options: + show_root_heading: true + show_source: false + +::: rglob.rglob.megabytes + options: + show_root_heading: true + show_source: false + +::: rglob.rglob.gigabytes + options: + show_root_heading: true + show_source: false + +::: rglob.rglob.terabytes + options: + show_root_heading: true + show_source: false + +!!! note "Path return at 2.0" + As of 2.0, `rglob()` / `rglob_()` return `list[Path]` (previously + `list[str]`). See [migrating to 2.0](migrating-to-2.0.md) for the + one-line migration. The new `find()` / `find_all()` family already + returns `Path` so code targeting the modern API doesn't need to change. + +[mkdocstrings]: https://mkdocstrings.github.io/ diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..d6cda6b --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,306 @@ +# Architecture + +This page describes the package shape and how the pieces fit together. It +accretes per phase — each later phase adds its own diagrams (CLI hierarchy, +walker call-graph, `dupes` pipeline, final public-API class diagram). + +## Phase 2 — Package layout + +```mermaid +graph TD + subgraph Source + SRC[src/rglob/] + INIT[__init__.py
Public API + __version__] + CORE[rglob.py
rglob, rglob_, lcount, tsize,
unit helpers] + CLI[cli.py
argparse → Typer in Phase 4] + PYTYPED[py.typed] + end + + subgraph Tests + PYTEST[tests/
pytest + hypothesis] + BEHAVE[features/
behave BDD] + BENCH[bench/
pytest-benchmark — Phase 5] + end + + subgraph Docs + MKDOCS[docs/
MkDocs Material] + ADRS[docs/decisions/
ADRs] + PLANS[docs/plans/
Modernization roadmap] + end + + subgraph CI + CIYML[.github/workflows/ci.yml
Lint · Test · Coverage] + RELYML[.github/workflows/release.yml
OIDC PyPI publish] + DOCSYML[.github/workflows/docs.yml
gh-pages deploy] + end + + SRC --> INIT + SRC --> CORE + SRC --> CLI + SRC --> PYTYPED + + PYTEST -. imports .-> SRC + BEHAVE -. imports .-> SRC + BENCH -. imports .-> SRC + + CIYML -. runs .-> PYTEST + CIYML -. runs .-> BEHAVE + DOCSYML -. builds .-> MKDOCS + RELYML -. builds .-> SRC +``` + +## Phase 2 — CI flow + +```mermaid +flowchart LR + PR[Pull Request] --> LINT[Lint job
ruff check · ruff format --check · mypy --strict] + PR --> TEST[Test matrix
3.11–3.14 × ubuntu/macos/windows] + TEST --> COV[pytest --cov
per-job fail_under=95] + TEST --> BDD[behave] + COV --> CC[Codecov upload
per matrix cell] + CC --> AGG[Codecov merged report
project gate = 100%] + LINT --> BP[Branch protection on master
blocks merge if any check fails] + AGG --> BP + BDD --> BP +``` + +## Walker call-graph — Phase 3 + +```mermaid +flowchart TD + USER[find(base, patterns, ...)] + COMPILE[_compile_patterns
fnmatch + ** translation] + HANDLER[_make_error_handler
ignore | warn | raise] + WALK[_walk
recursive descent] + SCAN[os.scandir
cached DirEntry.d_type] + SORT{sort?} + MATCH[_matches_any
basename or rel-path] + EXCLUDE[exclude matchers] + DEPTH{max_depth?} + HIDDEN{hidden=False
and name.startswith('.')?} + REAL[os.path.realpath
cycle memo] + YIELD[yield Path] + + USER --> COMPILE --> WALK + USER --> HANDLER --> WALK + WALK --> SCAN + SCAN --> SORT + SORT --> MATCH + MATCH --> EXCLUDE + EXCLUDE -- excluded --> WALK + EXCLUDE -- kept --> HIDDEN + HIDDEN -- skip --> WALK + HIDDEN -- keep --> YIELD + YIELD --> DEPTH + DEPTH -- under limit --> REAL + REAL -- new --> WALK + REAL -- already visited --> WALK +``` + +## Symlink-loop detection + +```mermaid +sequenceDiagram + participant F as find(...) + participant W as _walk + participant FS as os.scandir + participant R as os.path.realpath + participant V as visited: set[str] + + F->>V: add realpath(base) + F->>W: walk(base, depth=0) + W->>FS: scandir(base) + FS-->>W: [a/, sym->base] + W->>R: realpath(base/a/) + R-->>W: /base/a (new) + W->>V: add /base/a + W->>W: recurse(base/a, depth=1) + + W->>R: realpath(base/sym) + R-->>W: /base (already in V) + W-->>F: skip; do not recurse +``` + +## Pattern translation (the `**` story) + +```mermaid +flowchart LR + P[pattern: str] + HAS{contains **?} + SLASH{contains / or sep?} + TR1[fnmatch.translate
basename matcher] + TR2[fnmatch.translate
relpath matcher] + TR3["preprocess
**/ → (?:.*/)?
/** → (?:/.*)?
** → .*"] + REGEX[re.compile] + + P --> HAS + HAS -- no --> SLASH + SLASH -- no --> TR1 + SLASH -- yes --> TR2 + HAS -- yes --> TR3 --> REGEX + TR1 --> REGEX + TR2 --> REGEX +``` + +## CLI command hierarchy — Phase 4 + +```mermaid +graph TD + APP[rglob CLI
Typer + Rich] + + APP --> FIND[rglob find] + APP --> LCOUNT[rglob lcount] + APP --> TSIZE[rglob tsize] + APP --> COMPL[--install-completion] + + FIND --> FILTERS[Shared filter options
-E --exclude
-d --max-depth
-H --hidden
-L --follow
-s/-i --case-sensitive/-insensitive
--gitignore/--no-gitignore] + FIND --> FORMATS[Output formats
--json
--jsonl
-0 --null
--format TEMPLATE] + + LCOUNT --> LFLAGS[--no-empty
--no-comments] + + TSIZE --> TFLAGS[--unit kb/mb/gb/tb] + + classDef shared fill:#eef,stroke:#557,color:#000 + class FILTERS,FORMATS,LFLAGS,TFLAGS shared +``` + +Each subcommand calls into `rglob.rglob.find()` (or the legacy `rglob` / +`lcount` / `tsize` wrappers); the CLI is a thin Typer layer over the +library API. + +## `dupes` pipeline — Phase 5 + +```mermaid +flowchart LR + INPUT[Paths from find] + SIZE[Bucket by exact size] + HEAD[Bucket by xxh3_64
of first 4 KiB] + FULL[Bucket by blake2b
over full bytes] + OUT[Groups of 2+ paths] + + INPUT --> SIZE + SIZE -- "≥2 in bucket" --> HEAD + SIZE -- "1 in bucket" --> SKIP1[discard - unique] + HEAD -- "≥2 in bucket" --> FULL + HEAD -- "1 in bucket" --> SKIP2[discard - false-positive] + FULL -- "≥2 in bucket" --> OUT + FULL -- "1 in bucket" --> SKIP3[discard - confirmed unique] +``` + +Large unique files only ever read their first 4 KiB; the full hash only +runs on confirmed candidates. + +## `.gitignore` pruning sequence — Phase 5 + +```mermaid +sequenceDiagram + participant F as find(respect_gitignore=True) + participant G as gitignore_matcher + participant PS as pathspec.PathSpec + participant W as _walk + + F->>G: gitignore_matcher(base) + G->>G: rglob(".gitignore") + G->>PS: PathSpec.from_lines("gitignore", lines) + G-->>F: predicate(path) → bool + F->>W: walk(base, gitignore=predicate) + loop per entry + W->>G: predicate(path) + G->>PS: spec.match_file(rel) + PS-->>W: match? + W-->>W: skip if ignored, else yield/recurse + end +``` + +## Public API surface — 2.0 + +```mermaid +classDiagram + class rglob { + <> + +__version__: "2.0.0" + +find(base, patterns, *, exclude, max_depth, hidden, follow_symlinks, case_sensitive, sort, on_error, kinds, min_size, max_size, newer_than, older_than, newer_than_file, perm, uid, gid, respect_gitignore) Iterator~Path~ + +find_all(...) list~Path~ + +rglob(base, pattern) list~Path~ + +rglob_(pattern) list~Path~ + +lcount(base, pattern, func) int + +tsize(base, pattern, func) float + +kilobytes(value) float + +megabytes(value) float + +gigabytes(value) float + +terabytes(value) float + } + + class CLI { + <> + +find(patterns, ...) None + +grep(pattern, files_or_globs, ...) None + +count(patterns, ...) None + +lcount(pattern, ...) None + +tsize(pattern, ...) None + +stats(pattern, ...) None + +tree(pattern, ...) None + +top(pattern, ...) None + +dupes(pattern, ...) None + +describe(subcommand) None + +schema(subcommand) None + +capabilities() None + +agent_version() None + +mcp() None + } + + class agent { + <> + +__agent_api_version__: "1.0" + +search(WalkOptions) Iterator~FileMatch~ + +search_all(WalkOptions) FileSearchResult + +grep(GrepOptions) Iterator~LineMatch~ + +grep_all(GrepOptions) LineSearchResult + +count(CountOptions) Stats + +find_duplicates(WalkOptions) DuplicateSearchResult + } + + class _filters { + <> + +parse_size(value) int + +parse_time(value) datetime + +size_predicate(min_size, max_size) Callable + +mtime_predicate(newer_than, older_than) Callable + +kinds_match(entry, wanted) bool + +gitignore_matcher(base) Callable~|None + } + + class _dupes { + <> + +find_duplicates(paths) list~list~Path~~ + -_hash_head(path) str + -_hash_full(path) str + -_fast_hash(data) str + } + + CLI ..> rglob : calls + CLI ..> agent : structured JSON + rglob ..> _filters : uses + CLI ..> _dupes : uses +``` + +The human-facing `rglob` module stays compact, while `rglob.agent` is the +SemVer-locked integration surface for coding agents. Internal modules +(`_filters`, `_dupes`, `_grep`, `_count`) remain implementation details. + +## MCP request lifecycle + +```mermaid +sequenceDiagram + participant Host as MCP host + participant Server as rglob mcp + participant Agent as rglob.agent + participant Walker as find / grep / count + + Host->>Server: tool call (find_files, grep_content, ...) + Server->>Agent: build Options dataclass + Agent->>Walker: execute read-only search + Walker-->>Agent: matches, stats, or errors + Agent-->>Server: SearchResult / Stats dataclass + Server-->>Host: JSON-safe dict via to_json_dict() +``` diff --git a/docs/changelog.md b/docs/changelog.md new file mode 100644 index 0000000..9420640 --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1,26 @@ +# Changelog + +The canonical changelog lives at [`CHANGELOG.md`][src] in the repo root and +follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) / +[Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +[src]: https://github.com/chris-piekarski/python-rglob/blob/master/CHANGELOG.md + +## What's accreting toward 2.0 + +The 2.0 release accumulates across the six modernization phases described in +the [roadmap](plans/modernization-roadmap.md). At a glance: + +- **Phase 1** — Packaging hygiene: PEP 621 metadata, `src/` layout, single-sourced + `__version__`, `.gitignore`, refreshed docs. +- **Phase 2** — Tooling: Ruff/Mypy/Pytest, 100% aggregated coverage, MkDocs + Material, three GitHub workflows, `Makefile`. +- **Phase 3** — `find()` / `find_all()` on `os.scandir`, `**` globbing, + symlink-loop detection, hypothesis property tests. +- **Phase 4** — Typer + Rich CLI rewrite with filter flags and output formats. +- **Phase 5** — Fun features: `stats` / `tree` / `top` / `dupes`, + `.gitignore` awareness. +- **Phase 6** — Breaking change: `rglob()` / `rglob_()` return `list[Path]`. + Final v2.0.0 release. + +For per-phase detail, see [`CHANGELOG.md`][src] at the repo root. diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..c06d7a6 --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,140 @@ +# CLI Reference + +`rglob` ships a Typer-powered CLI with recursive search commands, +Rich-styled human output, structured JSON / JSONL / NUL-separated output +formats, and shell completion. Agent-oriented introspection endpoints always +emit plain JSON to stdout. + +## Auto-generated command tree + +::: mkdocs-typer2 + :module: rglob.cli + :name: app + +## Subcommand quick reference + +### `rglob find` + +Print paths under `--base` (default: CWD) matching one or more glob +patterns. Multiple positionals are OR'd. Filter and output options: + +| Flag | Meaning | +| ------------------------------------ | ------- | +| `-E, --exclude PATTERN` | Glob(s) to exclude *and* prune from descent. Repeatable. | +| `-d, --max-depth N` | Maximum recursion depth (0 = base only). | +| `-H, --hidden` | Include dotfiles and dot-directories. | +| `-L, --follow` | Follow symlinks (cycles are auto-detected). | +| `-s/-i, --case-sensitive/-insensitive` | Force case sensitivity (default: follow host OS). | +| `--json` | Emit a `FileSearchResult` object with `results`, truncation metadata, and errors. | +| `--jsonl` | Emit one compact `FileSearchResult` object line. | +| `--limit N` | Maximum structured records to emit. | +| `--max-bytes SIZE` | Maximum structured output bytes. | +| `--max-file-size SIZE` | Skip files larger than this. | +| `--include-errors/--no-include-errors` | Include per-record errors in structured output. | +| `-0, --null` | Separate paths with NUL bytes (for `xargs -0`). | +| `--format TEMPLATE` | Render each match through a mini-template. Available fields: `path`, `rel`, `name`, `size`, `size_kb`, `size_mb`. | + +### `rglob grep` + +```text +rglob grep [files-or-globs...] [--base DIR] [--json|--jsonl] +``` + +Searches matching files for a regex by default, or a literal with +`--fixed-string/-F`. Useful flags: + +| Flag | Meaning | +| ---- | ------- | +| `-i, --ignore-case` | Match content case-insensitively. | +| `-C, --context N` | Include N lines before and after each match. | +| `-B, --before N` / `-A, --after N` | Include asymmetric context. | +| `-m, --max-count N` | Stop after N matches. | +| `-w, --word` | Match whole words. | +| `-v, --invert` | Return non-matching lines. | +| `--encoding TEXT` | Decode files with this encoding. | +| `-a, --text` | Search binary files as text. | +| `--json` / `--jsonl` | Emit a `LineSearchResult`. | + +### `rglob count` + +```text +rglob count ... [--base DIR] [--no-empty] [--no-comments] [--json|--jsonl] +``` + +Reports files, lines, and bytes in one structured view. `--json` and +`--jsonl` emit the `Stats` schema used by the Python API and MCP tools. + +### `rglob lcount` + +```text +rglob lcount [--base DIR] [--no-empty] [--no-comments] +``` + +`--no-empty` skips blank lines; `--no-comments` skips `#`-prefixed lines. + +### `rglob tsize` + +```text +rglob tsize [--base DIR] [--unit {kb,mb,gb,tb}] +``` + +Defaults to `--unit mb`. Output is `Total size: X.XX UNIT`. + +### `rglob mcp` + +```text +pip install "rglob[mcp]" +rglob mcp +``` + +Starts the stdio MCP server. The server exposes `find_files`, +`grep_content`, `count_lines`, `find_duplicate_files`, and +`describe_subcommand`. + +## Quoting + +!!! warning "Quote your patterns" + Quote glob patterns so your shell doesn't pre-expand them. Use + `rglob find "*.py"`, not `rglob find *.py`. The CLI emits a warning to + stderr if it sees multiple positional arguments that look pre-expanded + (no glob metacharacters). + +## Shell completion + +```bash +rglob --install-completion bash # or zsh / fish / powershell +``` + +Then restart your shell. `` completes subcommands, flags, and option +values. + +## Agent introspection + +These commands are machine endpoints. They do not use Rich coloring and do +not require a TTY: + +```bash +rglob describe find +rglob schema find +rglob schema --all +rglob capabilities --json +rglob agent-version +``` + +- `describe ` returns arguments, options, default limits, and + input/output schemas. +- `schema ` returns only the input and output JSON Schema Draft + 2020-12 documents. +- `schema --all` returns every public JSON Schema document keyed by stable + schema stem. +- `capabilities --json` reports the agent API version, schema version, + installed extras, predicate support, and MCP availability. +- `agent-version` emits the current agent API version as a JSON string. + +## Exit codes + +| Code | Meaning | +| ---: | ------- | +| `0` | Success. | +| `2` | Invalid argument (e.g. unknown unit on `tsize`). | +| `1` | Unhandled error. | diff --git a/docs/contributing.md b/docs/contributing.md new file mode 100644 index 0000000..1fbcf3c --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1,37 @@ +# Contributing + +The canonical contributor guide lives at [`CONTRIBUTING.md`][src] in the repo +root. + +[src]: https://github.com/chris-piekarski/python-rglob/blob/master/CONTRIBUTING.md + +## Quick start + +```bash +git clone https://github.com/chris-piekarski/python-rglob.git +cd python-rglob +python -m venv .venv && source .venv/bin/activate +make dev-setup # installs [dev,bdd,docs,gitignore] and pre-commit hooks +make test # pytest + behave, gated at 100% local coverage +make lint # ruff + mypy --strict +``` + +## Standards we enforce + +- **Lint**: `ruff check` and `ruff format --check` must pass. +- **Types**: `mypy --strict src/rglob` must pass. +- **Tests**: `pytest --cov --cov-fail-under=100` and `behave` must pass. +- **Coverage**: 100% aggregated coverage on `src/rglob/` via Codecov merge + (see [ADR-0006](decisions/0006-aggregated-coverage.md)). Local runs enforce + 100%; CI per-job enforces 95% with the merged-report gate at 100%. + +CI runs the full matrix (Python 3.11–3.14 × Ubuntu/macOS/Windows). + +## Filing bugs + +Open an issue at . + +For security issues, please follow the [security policy][sec] (GitHub private +advisories). + +[sec]: https://github.com/chris-piekarski/python-rglob/security/policy diff --git a/docs/decisions/0001-build-backend.md b/docs/decisions/0001-build-backend.md new file mode 100644 index 0000000..61e53dc --- /dev/null +++ b/docs/decisions/0001-build-backend.md @@ -0,0 +1,30 @@ +# ADR-0001 · Build backend: `hatchling` + +**Status**: Accepted · **Date**: 2026-05-14 + +## Context + +The legacy `setup.py` is gone. PEP 621 metadata lives in `pyproject.toml`, +and we need a PEP 517 build backend that: + +- single-sources `__version__` from `src/rglob/__init__.py` +- supports a `src/` layout out of the box +- has zero plugin friction for typical packaging tasks (`sdist`/`wheel`) +- is well maintained in 2026 + +## Decision + +Use **`hatchling`** as the build backend, configured via +`[tool.hatch.build.*]` in `pyproject.toml`. Run builds with the `hatch` CLI +(`make build` → `hatch build`). + +## Consequences + +- `[tool.hatch.version] path = "src/rglob/__init__.py"` reads the version + string at build time — no manual sync between source and metadata. +- `[tool.hatch.build.targets.wheel] packages = ["src/rglob"]` makes the + `src/` layout work without extra glue. +- We get a low-noise, fast build with first-class PyPI OIDC compatibility. +- Alternatives considered: `setuptools` (more legacy baggage), `flit` (less + flexible for our metadata footprint), `pdm-backend` (good but + PDM-flavoured). diff --git a/docs/decisions/0002-python-floor.md b/docs/decisions/0002-python-floor.md new file mode 100644 index 0000000..c780b58 --- /dev/null +++ b/docs/decisions/0002-python-floor.md @@ -0,0 +1,30 @@ +# ADR-0002 · Python 3.11 floor + +**Status**: Accepted · **Date**: 2026-05-14 + +## Context + +The pre-2.0 codebase declared `python_requires=">=3.5"` despite using +features that wouldn't run cleanly on 3.5. The roadmap originally proposed a +**3.10** floor, but during the planning audit we noted that **3.10 reaches +upstream EOL in October 2026** — within months of when 2.0 is expected to +ship. + +## Decision + +Set `requires-python = ">=3.11"` for the 2.0 release. CI matrix runs +**3.11 → 3.12 → 3.13 → 3.14**. + +## Consequences + +- Two-month safety margin: 3.11 reaches EOL October 2027. +- We get to use: + - PEP 604 `X | Y` unions at runtime (no `from __future__ import annotations`) + - Native exception groups (`ExceptionGroup`, `except*`) + - `typing.Self` and `assert_type` + - `tomllib` in the stdlib + - `StrEnum` for legible CLI choices +- We drop: + - 3.10 users (`<1%` of `rglob` PyPI downloads at the time of writing) +- Lockstep with Astral / Pydantic / Polars / Hatch, all of which now floor + at 3.11+ or are moving there. diff --git a/docs/decisions/0003-path-return.md b/docs/decisions/0003-path-return.md new file mode 100644 index 0000000..94a1fa1 --- /dev/null +++ b/docs/decisions/0003-path-return.md @@ -0,0 +1,31 @@ +# ADR-0003 · `rglob` returns `list[Path]` at 2.0 + +**Status**: Accepted · **Date**: 2026-05-14 + +## Context + +Today `rglob.rglob()` returns `list[str]`. The Python ecosystem standardised +on `pathlib.Path` years ago; `os.scandir` (Phase 3's walker) is most +naturally consumed as `Path` instances; and downstream filters +(`min_size`, `--gitignore`) want `Path` semantics. The legacy `str` return +is a friction point in every example. + +## Decision + +At 2.0, `rglob()` and `rglob_()` return **`list[Path]`** instead of +`list[str]`. The new `find()` / `find_all()` family also yields/returns +`Path`. This is the **only intentional breaking change** in the 2.0 +release. + +## Consequences + +- One-line migration for users: + ```python + paths = [str(p) for p in rglob("/tmp", "*.txt")] + ``` +- Phase 6 includes a dedicated "Migrating to 2.0" docs page and CHANGELOG + call-out. +- Internally the wrappers stop doing `Path → str` coercion; everything is + `Path` end-to-end. +- We considered shipping a `rglob_str()` alias for migration but rejected + it — adds a forever-deprecated symbol for a one-line wrap. diff --git a/docs/decisions/0004-typer-rich.md b/docs/decisions/0004-typer-rich.md new file mode 100644 index 0000000..0b1efb1 --- /dev/null +++ b/docs/decisions/0004-typer-rich.md @@ -0,0 +1,33 @@ +# ADR-0004 · Typer + Rich for the CLI + +**Status**: Accepted · **Date**: 2026-05-14 + +## Context + +The pre-2.0 CLI uses stdlib `argparse`. We want: + +- Click-quality UX (subcommands, completion, friendly help) without raw + Click verbosity +- type-driven argument parsing (so the CLI surface tracks the typed API) +- pleasant terminal output (color, progress, tables) +- shell-completion for bash / zsh / fish / powershell out of the box + +## Decision + +Migrate to **Typer** (built on Click) with **Rich** for output formatting. + +- Typer for command/option declaration — type hints drive argument shape. +- Rich for tables (`stats`, `top`, `dupes`), trees (`tree`), and the + progress bar on long walks. + +## Consequences + +- Two new runtime dependencies. Both are widely deployed and stable. +- Shell completion is a one-liner per shell: + `rglob --install-completion {bash,zsh,fish,powershell}`. +- Docs page (`docs/cli.md`) uses **`mkdocs-typer2`** — the modern, + actively-maintained Typer plugin that renders the command tree directly + from the Typer app object. The older `mkdocs-click` bridge required a + manual `typer.main.get_command()` step and is no longer needed. +- We respect `NO_COLOR` and TTY detection; the snapshot tests run with + color disabled to keep diffs stable. diff --git a/docs/decisions/0005-single-release.md b/docs/decisions/0005-single-release.md new file mode 100644 index 0000000..dfb06fa --- /dev/null +++ b/docs/decisions/0005-single-release.md @@ -0,0 +1,34 @@ +# ADR-0005 · Single `2.0.0` release strategy + +**Status**: Accepted · **Date**: 2026-05-14 + +## Context + +The roadmap spans six phases. We considered cutting an intermediate release +(e.g. `2.0.0b1`) after each phase. That would: + +- ship value faster to early adopters +- catch packaging issues earlier on real installs +- give us tag-driven smoke-test coverage of `release.yml` + +…but it would also: + +- create six migration notes for users to chase +- pin us into supporting half-finished APIs (the Phase 3 walker before + Phase 4 wires the CLI to it, for example) +- conflict with the "labor of love" pacing — these are weekend phases + +## Decision + +**Ship 2.0.0 once, after all six phases have landed.** Source version +stays at PEP 440 `"2.0.0.dev0"` throughout development so editable installs +are visibly WIP. `release.yml` is wired in Phase 2 but only fires on +`v*` tag push. + +## Consequences + +- Phase 2's release-workflow verification is a **dry-run against + TestPyPI** rather than a real publish. +- One migration document, one CHANGELOG section, one PyPI version bump. +- If we change our mind mid-flight, we can always cut a `2.0.0a1` from a + partially-landed branch — no design lock-in here, just a default. diff --git a/docs/decisions/0006-aggregated-coverage.md b/docs/decisions/0006-aggregated-coverage.md new file mode 100644 index 0000000..c5deef8 --- /dev/null +++ b/docs/decisions/0006-aggregated-coverage.md @@ -0,0 +1,44 @@ +# ADR-0006 · Aggregated 100% coverage + +**Status**: Accepted · **Date**: 2026-05-14 + +## Context + +The original plan said "`fail_under = 100`" applied to every CI job. With a +matrix of **4 Python versions × 3 OSes = 12 cells**, that constraint is +brittle: + +- Phase 3 introduces `case_sensitive=None` (OS-dependent default), a + `realpath`-memoised symlink loop check (POSIX-leaning), and an + `on_error` branch for `PermissionError` on `/proc` (Linux-only). +- Forcing every cell to 100% pushes us toward sprinkling `# pragma: no + cover` on legitimate platform branches or writing tests with elaborate + `sys.platform` conditional skips. + +Codecov's project-status gate already merges coverage across all matrix +cells. That merged report is the source of truth a reviewer would actually +look at. + +## Decision + +- **Local** (`make test`): `--cov-fail-under=100`. Single-OS runs see the + full code surface (we always run on Linux locally), so 100% is achievable + and enforced. +- **CI per-job**: `--cov-fail-under=95`. A regression on any single + platform is loud but not catastrophic. +- **CI merged**: Codecov project status check enforces `target: 100%` + on the aggregated report. +- **`# pragma: no cover`** is reserved for genuinely-unreachable lines + (`if __name__ == "__main__":`, `if TYPE_CHECKING:`) and *cross-platform + guards that no matrix cell covers*. Each pragma must have a one-line + justification. + +## Consequences + +- A `if sys.platform == "win32":` branch is fine: covered by the Windows + cell, contributing to the merged 100%. +- A `if sys.platform == "haiku":` branch is not fine: no matrix cell + covers it. Either remove the branch or add a `# pragma: no cover — + Haiku not in matrix` with reasoning. +- Codecov configuration (`codecov.yml`) lives at repo root — to be added + alongside the Codecov badge. diff --git a/docs/decisions/0007-behave-parallel.md b/docs/decisions/0007-behave-parallel.md new file mode 100644 index 0000000..880c59c --- /dev/null +++ b/docs/decisions/0007-behave-parallel.md @@ -0,0 +1,37 @@ +# ADR-0007 · Behave kept as parallel BDD suite + +**Status**: Accepted · **Date**: 2026-05-14 + +## Context + +The pre-2.0 test suite is entirely Behave (BDD). The roadmap introduces +pytest as the primary suite in Phase 2. There were three plausible paths: + +1. **Delete Behave** — clean cut, single suite, lower maintenance. +2. **Keep Behave, deprecate it** — accept it as a transitional cost. +3. **Keep Behave as a first-class parallel suite** — both suites run on + CI, both gate merges. + +## Decision + +**Keep Behave** as a parallel BDD suite (option 3). The scenarios in +`features/rglob.feature` read like fun documentation (`Given I create 1100 +subdirectories…`) and that's worth preserving. + +To keep maintenance honest, Behave step definitions share helpers with +pytest: + +- `tests/conftest.py::TreeBuilder` is the canonical tree-building helper. +- `features/steps/steps.py` step bodies call into the same builder so any + new assertion shows up exactly once. + +## Consequences + +- Every new feature lands a pytest case **and** a Behave scenario if the + feature has user-visible behaviour. Tests-only features (e.g. internal + helpers) skip Behave. +- We **accept the drift risk**: in practice the Behave scenarios will rot + first if they diverge from the helpers. We treat that as a maintenance + bill rather than a blocker. +- The Behave job in CI runs alongside the pytest matrix and gates the same + way. diff --git a/docs/decisions/0008-security-model.md b/docs/decisions/0008-security-model.md new file mode 100644 index 0000000..393bbc6 --- /dev/null +++ b/docs/decisions/0008-security-model.md @@ -0,0 +1,58 @@ +# ADR-0008 · Security model + +**Status**: Draft (locked at Phase 3) · **Date**: 2026-05-14 + +## Context + +`rglob` walks filesystems. The Phase 3 walker rewrite adds: + +- `follow_symlinks=False` (default) → `True` (opt-in) symlink traversal +- a `realpath`-keyed memo to terminate symlink cycles +- `respect_gitignore` (Phase 5) reading `.gitignore` files top-down +- `on_error="ignore"|"warn"|"raise"` covering `PermissionError`, + `OSError`, `UnicodeDecodeError` + +Each of these is a small surface; together they are a security model worth +writing down so we don't drift. + +## Threat model + +**In scope** + +- *Symlink escape*: a follow-enabled walk should never visit a path whose + `realpath` lies outside the user-supplied `base`. Currently we **do not + enforce this** by default; users opt into `follow_symlinks=True` + knowing what it means. A future flag (`strict_base=True`) could enforce + containment. +- *Symlink loops*: a `realpath` memo terminates `a → b → a` and longer + cycles. The memo is scoped per `find()` call (no cross-call leaks) and + uses `os.path.realpath` (not `Path.resolve(strict=True)`) so dangling + links don't raise. +- *`.gitignore` containment*: `respect_gitignore` only reads + `.gitignore` files **at or below** `base`. It never traverses upward. + +**Out of scope** + +- *TOCTOU between `scandir` and `lstat`*: the walker uses cached + `DirEntry` metadata where possible; we do not snapshot the FS. A + malicious local user racing the walk can cause it to surface or miss + entries — this is inherent to filesystem walkers and not something we + can fix without a kernel-side lock. +- *Arbitrary code execution*: `rglob` never executes content it discovers. + Predicate callbacks (`lcount(..., func=...)`) run user code by design, + not by accident. + +## Mitigations + +- Default `follow_symlinks=False`. +- Per-call `realpath` memo for cycle termination. +- `respect_gitignore` traversal is `base`-rooted, never upward. +- `on_error="warn"` (default) prints to `stderr` rather than terminating + on unreadable entries; users opt into `"raise"` for strict pipelines. +- `tests/test_symlinks.py` builds a cycle and asserts termination. + +## Reporting + +Vulnerability reports go via the project's +[security policy](https://github.com/chris-piekarski/python-rglob/security/policy) +(GitHub private security advisories). diff --git a/docs/decisions/0009-agent-api-contract.md b/docs/decisions/0009-agent-api-contract.md new file mode 100644 index 0000000..77ba7a9 --- /dev/null +++ b/docs/decisions/0009-agent-api-contract.md @@ -0,0 +1,110 @@ +# ADR-0009 - Agent API contract + +**Status**: Accepted - **Date**: 2026-05-15 + +## Context + +The 2.0 release adds agent-facing search surfaces: + +- bounded JSON and JSONL CLI output +- a typed `rglob.agent` Python namespace +- a stdio MCP server +- machine-readable `describe`, `schema`, `capabilities`, and + `agent-version` CLI endpoints + +Human-oriented help text is not enough for autonomous tools. Agents need +stable field names, error envelopes, truncation metadata, and schemas that +can be inspected without scraping Rich-rendered output. + +## Decision + +`rglob` will expose a focused agent contract with its own semantic version: +`__agent_api_version__ = "1.0"`. The package version stays `2.0.0`, but +the agent contract can move independently in future releases. + +Only these surfaces are covered by the agent compatibility promise: + +- public dataclasses and function signatures exported from `rglob.agent` +- JSON Schema documents emitted by `rglob.agent.schema_for`, + `rglob.agent.all_schemas`, and `rglob schema` +- structured CLI subcommands and flags used by agents +- structured CLI JSON and JSONL output shapes +- MCP tool names, input shapes, and output shapes +- error codes, error envelopes, truncation metadata, and capability reports + +Private helper modules such as `_filters`, `_dupes`, and walker internals +remain implementation details. + +Agent results use concrete frozen, slotted dataclasses rather than a generic +wrapper. The v1 contract includes `FileSearchResult`, +`LineSearchResult`, and `DuplicateSearchResult`, each with explicit +metadata fields: + +- `results` +- `truncated` +- `total_files_searched` +- `bytes_read` +- `errors` +- `truncated_reason` + +`Path`, `datetime`, and `StrEnum` are allowed in Python dataclasses for +ergonomic typed use. Wire output is normalized through a single +`to_json_dict()` helper: + +- `Path` serializes to string. +- Absolute `path` fields use the host filesystem spelling. +- `relative_path` is POSIX-style relative to the requested base and never + starts with `./`. +- `datetime` serializes as ISO 8601 UTC with a `Z` suffix. +- `ErrorCode` serializes as its string value. + +Structured failures use one envelope: + +```json +{"ok": false, "error": {"code": "REGEX", "message": "...", "path": null}} +``` + +Operational issues discovered during a search are represented as +`ErrorInfo` records when the agent API or MCP server is configured to +collect errors. The legacy public `find(on_error=...)` warning and raising +behavior is preserved for compatibility. + +`ErrorCode` is a closed `StrEnum` for v1.x: + +- `PERM` +- `UNREADABLE` +- `BINARY` +- `TIMEOUT` +- `REGEX` +- `BAD_PREDICATE` +- `UNSUPPORTED_PLATFORM` + +JSON Schema documents use Draft 2020-12, include an `$id`, include a +`version` field, and are generated directly from the dataclasses in +`src/rglob/agent/_models.py`. `rglob.agent.schema_for()` +returns one public schema, `rglob.agent.all_schemas()` returns the full +mapping, `rglob schema ` returns a command's input/output +schemas, and `rglob schema --all` emits every public schema for callers +that want a file artifact. + +Compatibility rules: + +- Adding a dataclass field, CLI flag, MCP tool field, or error code is a + minor bump of `__agent_api_version__`. +- Removing or renaming any locked field, flag, function, or tool is a major + bump. +- Patch releases may clarify descriptions and fix bugs without changing the + emitted schema shape. +- While `__agent_api_version__` starts with `"1."`, `rglob` will emit + schemas that validate against the v1.x contract. + +## Consequences + +Agents can pin to `rglob.agent` and the schema version instead of inferring +behavior from human help text. The cost is that every future change to the +agent-facing surface must be reviewed as a compatibility decision. + +Tests must compare the Python API, CLI JSON/JSONL, MCP outputs, and +generated schemas against the same golden fixture records. `_models.py` is +the single source of truth, so committed schema artifacts cannot drift from +the contract. diff --git a/docs/decisions/0010-agent-safety-model.md b/docs/decisions/0010-agent-safety-model.md new file mode 100644 index 0000000..2ecc150 --- /dev/null +++ b/docs/decisions/0010-agent-safety-model.md @@ -0,0 +1,88 @@ +# ADR-0010 - Agent safety model + +**Status**: Accepted - **Date**: 2026-05-15 + +## Context + +Coding agents can call tools repeatedly, pass broad paths, and consume large +machine-readable outputs. `rglob` remains read-only, but search still has +safety risks: + +- walking outside the intended base through symlinks +- exposing file contents through grep +- dumping unbounded results over stdout or stdio +- hanging on large trees or slow filesystem calls +- failing ambiguously on permissions, binary files, bad predicates, or + unsupported platform-specific filters + +The existing public walker is human-friendly and backward-compatible. The +agent and MCP surfaces need stricter defaults. + +## Decision + +The agent surfaces are read-only. They list files, read file metadata, read +file contents for grep/count operations, and hash file contents for +duplicate detection. They never edit, delete, execute, or chmod files. + +Agent and MCP defaults are conservative: + +- `strict_base=True` +- `follow_symlinks=False` +- `include_errors=True` +- bounded `limit` +- bounded `max_bytes` +- bounded `max_file_size` +- cooperative `timeout_seconds` +- `.gitignore` respected only when requested or documented as an agent + default for the specific surface + +CLI path streaming can remain unbounded for human workflows. Structured +CLI output must always include truncation metadata when it uses the agent +record model. + +Path containment: + +- `strict_base=True` means every emitted path must remain under the + requested base after normalization. +- Symlink traversal is opt-in. When enabled under strict mode, symlink + targets outside the base are reported as errors instead of being emitted. +- `.gitignore` discovery is base-rooted and never walks upward. + +Content disclosure: + +- Grep and count read file contents by design. +- Binary files are skipped by default for grep and reported with `BINARY` + when errors are included. +- `--text` / `text=True` is the explicit opt-in for treating binary files + as text. +- Decoding defaults to UTF-8 with replacement unless the caller supplies + another encoding. + +Output exhaustion: + +- `limit` caps result records. +- `max_bytes` caps bytes read or emitted, depending on the operation. +- `max_file_size` skips individual files that are too large for the + requested operation. +- Truncated results set `truncated=True` and provide + `truncated_reason`. + +Timeouts are cooperative only. The walker checks `time.monotonic()` during +directory iteration and at yielded matches. Grep and duplicate detection +check between files and read chunks. No signals or worker-thread +cancellation are used. A single slow filesystem call may run past the +deadline. + +Errors are machine-readable. User mistakes and operational failures use +`ErrorInfo` records or the stable error envelope from ADR-0009 rather than +only colored stderr. + +## Consequences + +Agent integrations get predictable failure modes and bounded output by +default. Existing human-facing APIs keep their compatibility behavior, +which means the agent implementation must adapt the walker instead of +breaking `find(on_error=...)`. + +Tests must cover truncation, timeouts where practical, unreadable paths, +binary grep behavior, bad predicates, and strict-base symlink escapes. diff --git a/docs/decisions/index.md b/docs/decisions/index.md new file mode 100644 index 0000000..e9044bf --- /dev/null +++ b/docs/decisions/index.md @@ -0,0 +1,18 @@ +# Decisions Log (ADRs) + +Lightweight Architecture Decision Records capturing the choices locked in by +the [modernization roadmap](../plans/modernization-roadmap.md). Each ADR +follows a slim format: **Context → Decision → Consequences**. + +| # | Title | Status | +| ---- | ------------------------------------------------- | -------- | +| 0001 | [Build backend: hatchling](0001-build-backend.md) | Accepted | +| 0002 | [Python 3.11 floor](0002-python-floor.md) | Accepted | +| 0003 | [`rglob` returns `list[Path]` at 2.0](0003-path-return.md) | Accepted | +| 0004 | [Typer + Rich for the CLI](0004-typer-rich.md) | Accepted | +| 0005 | [Single 2.0.0 release strategy](0005-single-release.md) | Accepted | +| 0006 | [Aggregated 100% coverage](0006-aggregated-coverage.md) | Accepted | +| 0007 | [Behave kept as parallel BDD suite](0007-behave-parallel.md) | Accepted | +| 0008 | [Security model](0008-security-model.md) | Draft | +| 0009 | [Agent API contract](0009-agent-api-contract.md) | Accepted | +| 0010 | [Agent safety model](0010-agent-safety-model.md) | Accepted | diff --git a/docs/examples/agent-recipes.md b/docs/examples/agent-recipes.md new file mode 100644 index 0000000..f4a2e6f --- /dev/null +++ b/docs/examples/agent-recipes.md @@ -0,0 +1,67 @@ +# Agent Recipes + +## Find Python files modified recently + +```bash +rglob find "*.py" --newer-than 7d --json +``` + +## Grep TODOs with context + +```bash +rglob grep TODO "*.py" --context 2 --exclude .venv --exclude dist --json +``` + +## Count non-empty Python lines + +```bash +rglob count "*.py" --no-empty --no-comments --json +``` + +## Find duplicate downloads over 1 MiB + +```bash +rglob dupes "*" --base ~/Downloads --min-size 1MiB +``` + +--- + +## Runnable smoke tests + +These blocks exercise the same surface against the project's golden +fixture (`tests/fixtures/agent-tree/`). They run as part of `make test` +via `tests/test_examples.py`, so the published recipes can't silently +rot. The fixture path is injected as the `$RGLOB_FIXTURE` env var. + + +```bash +rglob find "*.py" --base "$RGLOB_FIXTURE" --json +``` + + +```bash +rglob grep TODO "*.py" --base "$RGLOB_FIXTURE" --context 1 --json +``` + + +```bash +rglob schema find +``` + + +```bash +rglob capabilities --json +``` + + +```python +import os +from pathlib import Path + +from rglob.agent import WalkOptions, search_all + +fixture = Path(os.environ["RGLOB_FIXTURE"]) +result = search_all(WalkOptions(patterns=["*.py"], base=fixture, limit=100)) +for match in result.results: + print(match.relative_path) +``` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..8efded5 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,46 @@ +# rglob + +Lightweight recursive glob helpers for Python — find files, count lines, sum +sizes. + +This is a small labor-of-love package. Modern Python has `pathlib.Path.rglob` +and tools like `fd`/`ripgrep` exist; `rglob` is small, easy to read, and ships +with friendly helpers for the two things you usually want next: count lines +and sum sizes. + +## Installation + +```bash +pip install rglob +``` + +## Quick start + +```python +import rglob + +files = rglob.rglob("/path/to/project", "*.py") +files_cwd = rglob.rglob_("*.py") + +lines = rglob.lcount( + "/path/to/project", + "*.py", + lambda line: bool(line.strip()) and not line.lstrip().startswith("#"), +) + +mb = rglob.tsize("/path/to/photos", "*.jpg", rglob.megabytes) +``` + +The full modern API (`find` / `find_all` with filter flags) is described on +the [API page](api.md). For the command-line, see [CLI](cli.md). For design +context, see [Architecture](architecture.md) and the [decisions log](decisions/index.md). + +## Where to go next + +- [API reference](api.md) +- [CLI reference](cli.md) +- [Architecture](architecture.md) — package layout, walker call-graph +- [Decisions log](decisions/index.md) — ADRs for locked-in design choices +- [Modernization roadmap](plans/modernization-roadmap.md) — the six-phase plan + currently shipping into 2.0 +- [Changelog](changelog.md) diff --git a/docs/migrating-to-2.0.md b/docs/migrating-to-2.0.md new file mode 100644 index 0000000..56e62de --- /dev/null +++ b/docs/migrating-to-2.0.md @@ -0,0 +1,87 @@ +# Migrating to 2.0 + +`rglob` 2.0 is a top-to-bottom modernization (see the +[roadmap](plans/modernization-roadmap.md)). The vast majority of changes are +additive — new helpers, new CLI flags, better docs. **One** intentional +breaking change is worth calling out: + +## The breaking change: `list[Path]` return type + +`rglob()` and `rglob_()` now return **`list[pathlib.Path]`** instead of +`list[str]`. The new `find()` / `find_all()` family already returns `Path`, +so code that already migrated to them is unaffected. + +### One-line migration + +```diff +- paths: list[str] = rglob("/repo", "*.py") ++ paths: list[Path] = rglob("/repo", "*.py") # if you want Path ++ paths: list[str] = [str(p) for p in rglob("/repo", "*.py")] # if you want str +``` + +### Why this changes + +- Modern Python (`pathlib.Path`, `os.PathLike`, `mypy` strict mode) is much + nicer to use with `Path` instances than raw strings. +- The Phase 3 walker rewrite is on `os.scandir`, which natively produces + `Path`-compatible entries. +- Downstream filters introduced in Phase 5 (`--gitignore`, `min_size`, + `kinds=...`) want `Path` semantics anyway. + +See [ADR-0003](decisions/0003-path-return.md) for the full reasoning. + +## Other changes worth knowing + +These are not breaking, but they are new defaults you might encounter: + +### Sorted output + +`rglob()` and `find()` now sort entries by default (the 1.x walker +documented its output as "not guaranteed sorted"). Pass `sort=False` to opt +out: + +```python +paths = rglob.find_all("/repo", "*.py", sort=False) +``` + +### Python floor: 3.11 + +`rglob` 2.0 requires **Python 3.11+** (3.10 reaches upstream EOL in October +2026; see [ADR-0002](decisions/0002-python-floor.md)). Pin `rglob<2` if you +need 3.10 support: + +```text +rglob<2; python_version<"3.11" +rglob>=2; python_version>="3.11" +``` + +### `**` patterns now work + +The 1.x README explicitly disclaimed `**` support +(`"**/*.py" is not supported here`). The 2.0 walker handles `**` natively: + +```python +rglob.find_all("/repo", "**/*.py") +``` + +### Symlink handling is opt-in + +By default, the walker does **not** follow symbolic links. Pass +`follow_symlinks=True` to opt in; cycles are terminated automatically via a +realpath memo. See [ADR-0008](decisions/0008-security-model.md). + +### Hidden files are skipped by default + +Dotfiles and dot-directories are skipped unless you pass `hidden=True`. + +## Doing a clean upgrade + +1. Bump your pin to `rglob>=2,<3`. +2. Run your test suite — type checkers will flag any `list[str]` callers. +3. Apply the `str(p)` wrap where you really need strings, or migrate to + `Path` semantics. +4. Adopt `find()` / `find_all()` for new code — they have the full filter + suite (`exclude`, `max_depth`, `hidden`, `kinds`, `min_size`, etc.). + +If you hit anything surprising, please file an issue at +. diff --git a/docs/plans/agent-platform.md b/docs/plans/agent-platform.md new file mode 100644 index 0000000..2945f17 --- /dev/null +++ b/docs/plans/agent-platform.md @@ -0,0 +1,793 @@ +# `python-rglob` 2.0 — Agent Platform Additions + +## Context + +This plan **extends the in-flight 2.0 PR ([#6](https://github.com/chris-piekarski/python-rglob/pull/6))**; +it does not gate a new release. The goal is to position `rglob` as the +**default recursive-search dependency for coding agents** (Claude Code, +Cursor, Aider, Gemini CLI, Codex CLI, generic MCP clients) for +**filename globbing**, **content grep**, and **`find(1)`-style +predicates** — exposed through a stability-promised, agent-friendly +surface. + +The audience widens from "human writing scripts" to "autonomous agent +shelling out, importing, or talking MCP". Every phase below serves that +audience and ships as **additional commits on `release/2.0`**. + +**Decisions locked in**: + +- **Version**: stays at `2.0.0`. Single-release strategy + ([ADR-0005](../decisions/0005-single-release.md)) still applies — we + ship one PyPI publish covering everything from the modernization + roadmap *and* this plan. The `CHANGELOG.md` `[2.0.0]` section already + exists on `release/2.0`; it accumulates the agent additions and its + date is refreshed at tag time if needed. +- **Scope**: glob + grep + find-style predicates. Pure Python; no + vendored ripgrep / GNU find. Goal is a credible agent-default for the + *common* 90% of search tasks, with bounded outputs and predictable + failure modes, not a ripgrep replacement at the perf frontier. +- **Integration**: three first-class surfaces — (a) stable bounded JSON CLI, + (b) typed `rglob.agent` Python API, (c) `rglob mcp` stdio MCP server. + No HTTP/OpenAPI in 2.0 (revisit if demand emerges). +- **Stability promise**: a focused `rglob.agent` namespace — *only* its + dataclasses, function signatures, JSON schemas, MCP tool list, and + CLI subcommand+flag surface are SemVer-locked. Error envelopes, + truncation metadata, resource-limit semantics, and capability reports + are part of that contract. Everything else (`_filters`, `_dupes`, + walker internals) stays free to evolve. +- **Discovery**: `AGENTS.md` grows a "consumer-facing" half (today it + only speaks to *contributors*); `docs/examples/` ships runnable + recipes; `rglob describe ` and `rglob schema ` + give agents a machine-readable manifest without scraping `--help`; + `rglob schema --all` emits the full schema set for artifact-oriented + consumers; `rglob capabilities --json` reports + installed extras, platform-supported predicates, MCP availability, and + schema/API versions. + +**Out of scope for 2.0** (revisit later): + +- OpenAPI / JSON-RPC over HTTP +- `llms.txt` manifest (re-evaluate when the spec is more settled) +- Replacing ripgrep on huge corpora (we don't compete on raw GBs/sec) +- Symbolic search (LSP-style "find references") +- Editing operations (we stay read-only) +- A full Git-compatible `.gitignore` engine beyond the practical + `pathspec`-based behaviour already in 2.0 + +**Prerequisite**: the 2.0 PR's lone outstanding CI failure +(`test_find_help_lists_filter_flags`, an ANSI-stripping flake on +GitHub Actions' `FORCE_COLOR=1` env) needs a 5-line fix on +`release/2.0` before agent-platform commits start landing. The patch is +already in the working tree and should be committed before this plan's +first implementation commit. + +--- + +## Implementation checklist + +This section is the durable handoff log for agent-platform work. Keep it +updated as changes land so another coding agent can restart from the +current state without reconstructing context from git history. + +**Status legend**: `[ ]` not started, `[~]` in progress, `[x]` done, +`[!]` blocked / needs attention. + +### Current status + +- `[~]` Overall phase: Agent platform implementation is locally green in + the working tree, but not committed yet. +- `[!]` Repository state: the agent-platform implementation and schema + cleanup are currently unstaged/untracked working-tree changes on + `release/2.0`; stage/commit/push them before treating the PR as updated. +- `[x]` Prerequisite help flake patch exists in the working tree and + passes `.venv/bin/pytest -q tests/test_cli.py::test_find_help_lists_filter_flags`. +- `[x]` Phase 0a ADRs 0009/0010 written and linked from docs navigation. +- `[x]` Phase 0b contract dataclasses and `tests/fixtures/agent-tree/` + added to the working tree. +- `[x]` Phase 0c runtime schema generation wired from `_models.py`; no + committed schema cache or drift tooling remains. +- `[x]` Phase 0d CLI introspection commands and structured JSON output + implemented. +- `[x]` Docs, changelog, and snapshots updated for Phase 0. +- `[x]` Phase 1 grep engine, `grep` CLI, `count` CLI, structured JSON, + and additional find predicates. +- `[x]` Phase 2 public `rglob.agent` API. +- `[x]` Phase 3 `rglob mcp` stdio MCP server and optional extra. +- `[x]` Phase 4 AGENTS.md consumer half, `docs/agents/*`, and README + top-fold rewrite. +- `[~]` Phase 4 `docs/examples/` content exists, but the runnable examples + harness `tests/examples_harness.py` is not implemented yet. +- `[~]` Phase 5 packaging/docs polish is complete for the implemented + surface; final release/tag work remains pending. +- `[x]` Review follow-up verification: import cycle, strict base + containment, byte-limit prechecks, and timeouts fixed; final build gate + passed. +- `[x]` Final verification run: `make lint`, `make test`, `make build`. +- `[ ]` Add `tests/examples_harness.py` and wire runnable docs examples + into CI / `make test`. +- `[ ]` Stage and commit the current working-tree changes with a + Conventional Commit subject, then push/update the pull request. +- `[ ]` Tag `v2.0.0` after PR merge/release approval so `release.yml` + publishes to PyPI and creates the GitHub release. + +### Work log + +- 2026-05-15: Began implementation pass. Added this checklist so progress + survives context compaction or agent handoff. +- 2026-05-15: Completed Phase 0a. Added ADR-0009 (agent API contract), + ADR-0010 (agent safety model), decisions index entries, and MkDocs nav. +- 2026-05-15: Completed Phase 0b/0c in the working tree. Added + `src/rglob/agent/_models.py`, the golden fixture tree, runtime schema + generation in `src/rglob/agent/_introspection.py`, and schema tests. +- 2026-05-15: Completed Phase 0d in the working tree. Added `describe`, + `schema`, `capabilities`, and `agent-version`; changed `find --json` and + `find --jsonl` to emit `FileSearchResult`; updated CLI docs and changelog. + `make test` passes with 100% coverage and Behave green. +- 2026-05-15: Completed Phase 1 in the working tree. Added `grep` and + `count` helpers/CLI commands, structured `LineSearchResult`/`Stats` JSON, + `perm`/`uid`/`gid`/`newer_than_file` filters, runtime-generated schemas, + and tests. `make test` passes with 100% coverage and Behave green. +- 2026-05-15: Completed Phase 2 in the working tree. Added public + `rglob.agent` exports and non-raising `search`, `search_all`, `grep`, + `grep_all`, `count`, and `find_duplicates` functions with tests. +- 2026-05-15: Completed Phase 3 in the working tree. Added `rglob[mcp]`, + `src/rglob/agent/mcp.py`, `rglob mcp`, and fake-SDK tests covering the + stdio server registration and JSON result shapes. `make test` passes with + 100% coverage and Behave green. +- 2026-05-15: Completed documentation and packaging polish for the + implemented working-tree surface. Updated README, AGENTS consumer + guidance, CLI docs, architecture, changelog, `docs/agents/`, and + `docs/examples/`. `tests/examples_harness.py` remains pending. +- 2026-05-15: Addressed review findings in the working tree. Removed the + fresh-process CLI import cycle, enforced strict base containment for + symlink escapes, moved `max_bytes` checks ahead of file body reads, and + wired `timeout_seconds` through search, grep, count, and duplicate + helpers. Added regression tests for each issue. Verification passed: + `make lint`, `make test`, `make build`, and `make docs-build`. +- 2026-05-15: Cleaned up schema generation after review. Removed committed + `_schemas/*.json`, removed `scripts/generate_agent_schemas.py`, removed + schema drift checks from Makefile/CI, generated schemas on demand from + `_models.py`, added `rglob schema --all`, and verified `make lint`, + `make test`, and `make build`. +- 2026-05-15: Updated this checklist to distinguish implemented + working-tree changes from committed/PR state. Pending items are the + docs examples harness, staging/committing/pushing the PR changes, and + the eventual `v2.0.0` tag/release. + +--- + +## Phase 0 — Contract, safety, and golden fixtures + +Do this before expanding the surface. Agents need stable machine +contracts more than they need another flag, and the current CLI JSON +already has two shapes (`find --json` is a string array; `find --jsonl` +is object records). Lock the record model first so Phase 1 does not +ship unstable JSON by accident. + +**Sub-phase order** (each chunk lands as its own conventional commit on +`release/2.0`): + +- **0a** — Write ADRs 0009 (contract) and 0010 (safety). No code yet. +- **0b** — Land `src/rglob/agent/_models.py` and the + `tests/fixtures/agent-tree/` golden fixture. +- **0c** — Generate JSON Schema Draft 2020-12 documents on demand from + `src/rglob/agent/_models.py`. +- **0d** — Add the four Typer subcommands (`describe`, `schema`, + `capabilities`, `agent-version`); update existing subcommands to + produce the new structured JSON. + +**Tasks** + +- New ADRs: + - `docs/decisions/0009-agent-api-contract.md` — SemVer rules for + `rglob.agent`, CLI JSON, schemas, and MCP tools. + - `docs/decisions/0010-agent-safety-model.md` — read-only threat + model for agents and MCP: path containment, symlink escape, + content disclosure, binary files, unreadable files, and output + exhaustion. +- Define the shared contract models in `src/rglob/agent/_models.py` + before implementing the public API: `FileMatch`, `LineMatch`, `Stats`, + `Duplicate`, `ErrorInfo`, `ErrorCode`, `FileSearchResult`, + `LineSearchResult`, `DuplicateSearchResult`, `CapabilityReport`, + `WalkOptions`, `GrepOptions`, `CountOptions`. All are + `@dataclass(frozen=True, slots=True)` (no Pydantic). We use **three + concrete result classes** rather than a `Generic[T]` wrapper — frozen+ + slotted dataclasses with `Generic[T]` are awkward, and one schema per + result type is exactly what agents want. Minimum viable field shapes + (refined during ADR-0009 drafting): + + - `WalkOptions`: `patterns`, `base`, `exclude`, `max_depth`, `kinds`, + `min_size`/`max_size`, `newer_than`/`older_than`, `limit`, + `max_bytes`, `max_file_size`, `timeout_seconds`, `strict_base`, + `follow_symlinks`, `respect_gitignore`, `include_errors`, + `case_sensitive`, `sort`. + - `FileMatch`: `path: Path`, `relative_path: str`, + `size: int`, `mtime: datetime`, `kinds: list[Kind]`, + `errors: list[ErrorInfo]`. `kinds` is plural because an executable + file is both `"f"` and `"x"`, and a symlink can have its own tag. + - `LineMatch`: `path: Path`, `line_number: int`, `content: str`, + `before: list[str]`, `after: list[str]`, `encoding: str`. + - `FileSearchResult` / `LineSearchResult` / `DuplicateSearchResult`: + each carries `results: list[]`, `truncated: bool`, + `total_files_searched: int`, `bytes_read: int`, + `errors: list[ErrorInfo]`, `truncated_reason: str | None`. A common + metadata mixin can DRY the shared fields, but the public types stay + concrete. + - `ErrorInfo`: `code: ErrorCode` (a `StrEnum` — see below), + `message: str`, `path: Path | None`. + - `ErrorCode`: a closed `StrEnum` for the v1.x agent API: + `PERM`, `UNREADABLE`, `BINARY`, `TIMEOUT`, `REGEX`, + `BAD_PREDICATE`, `UNSUPPORTED_PLATFORM`. ADR-0009 records that + *adding* a code is a minor `__agent_api_version__` bump; renaming + or removing one is a major bump. + - `CapabilityReport`: locked shape: + + ```json + { + "agent_api_version": "1.0", + "schema_version": "1.0", + "package_version": "2.0.0", + "extras": {"mcp": true, "gitignore": true, "ext": false}, + "predicates": { + "perm": "supported", + "uid": "POSIX-only", + "gid": "POSIX-only", + "newer_than": "supported", + "respect_gitignore": "supported" + }, + "mcp": {"available": true, "transport": "stdio"} + } + ``` + +- **Wire serialization is locked in ADR-0009**: + - Python dataclasses may store `Path`, `datetime`, and `StrEnum` + instances for ergonomic typed use. + - JSON / JSONL / schema / MCP wire output serializes every `Path` as + a string. `path` is an absolute normalized path using the host + filesystem spelling; `relative_path` is POSIX-style relative to the + requested `base` and never starts with `./`. + - `datetime` fields serialize as ISO 8601 UTC strings with a `Z` + suffix. + - `ErrorCode` serializes as its string value. + - A shared `to_json_dict()` helper in `src/rglob/agent/_models.py` + handles this conversion for CLI JSON, schema examples, and MCP so + the three surfaces cannot drift. + +- Every option model includes resource controls: `limit`, + `max_bytes`, `max_file_size`, `timeout_seconds`, `strict_base`, + `follow_symlinks`, `respect_gitignore`, and `include_errors`. + MCP defaults are conservative (`limit` set, `strict_base=True`, + `follow_symlinks=False`); CLI streaming remains human-friendly, but + structured `--json` / `--jsonl` always reports whether output was + truncated. + `include_errors=True` (the agent/MCP default) makes the walker collect + `ErrorInfo` records instead of emitting `RuntimeWarning` or raising; + the public `find(on_error=...)` API and its warning behaviour are + unchanged for backward compatibility. +- **`timeout_seconds` semantics** — *cooperative only*. The walker + checks `time.monotonic()` at every `scandir()` iteration and at every + yielded match; on expiry it raises `TIMEOUT` via the error envelope + (or `ErrorInfo` when `include_errors=True`) and returns whatever was + collected so far in the `SearchResult`. Minimum value is 0.1s; a + finer floor isn't useful for filesystem walks. We do **not** use + signals (POSIX-only) or threads (cancellation is hard). Document the + precision honestly: the walker can run a bit past the deadline if a + single `scandir()` call is slow. +- Define the stable error envelope once: + `{"ok": false, "error": {"code": "...", "message": "...", "path": ...}}`. + User mistakes (bad regex, bad predicate, unsupported platform field) + must be machine-readable, not only Rich-coloured stderr. +- Add a tiny golden fixture tree under `tests/fixtures/agent-tree/` + (committed, < 50 files). All three surfaces (Python API, CLI `--json`/`--jsonl`, + and MCP tools) must produce identical `SearchResult` / `FileMatch` / + `LineMatch` records against it. Required structure: + + ``` + tests/fixtures/agent-tree/ + .gitignore + src/ + main.py + utils/ + helper.py + docs/ + notes.txt + binary.bin # non-UTF-8 content + unreadable/ # permission-denied case (mocked or chmod 000) + duplicates/ + a.txt + b.txt (byte-identical copy) + hidden/ + .secret.py + ``` + + Tests exercise hidden entries, gitignore pruning, binary skipping, + context capture, truncation, and error envelopes. The fixture must + produce identical `FileSearchResult` / `LineSearchResult` / + `DuplicateSearchResult`, `FileMatch`, and `LineMatch` records + across the Python API, CLI JSON, and MCP surfaces. +- Add schema tooling without a runtime modelling dependency. **Decision**: + keep `src/rglob/agent/_models.py` as the sole source of truth and + generate JSON Schema Draft 2020-12 documents on demand in + `src/rglob/agent/_introspection.py`. The generator is stdlib-only and + powers `rglob.agent.schema_for()`, `rglob.agent.all_schemas()`, + `rglob schema `, and `rglob schema --all`. +- **No schema cache** — generated JSON files are not committed or packaged. + This removes the `make schemas` target, the `make build` drift check, + and the CI schema-drift step because there is no second artifact to keep + synchronized. +- **Generator contract** (for Phase 0c): schema generation imports the + dataclasses from `rglob.agent._models`, emits one JSON Schema Draft + 2020-12 document per public type, includes stable `$id` and `version` + fields, and remains deterministic. +- **Chosen introspection CLI surface** (locked for implementation): + - `rglob describe ` — full JSON manifest (arguments, types, + defaults, input/output schemas, default limits, supported extras). + - `rglob schema ` — just the input + output JSON Schema + objects. + - `rglob schema --all` — every public JSON Schema object, keyed by stable + schema stem. + - `rglob capabilities --json` — reports `__agent_api_version__`, installed + extras, platform-supported predicates, MCP availability, and schema + versions. + - `rglob agent-version` — prints the current `__agent_api_version__`. + These are implemented as normal Typer subcommands (no global-option + parsing hacks). All existing subcommands (`find`, `grep`, `count`, + `lcount`, `tsize`, `stats`, `tree`, `top`, `dupes`) receive `describe` + and `schema` support. + **`describe` / `schema` / `capabilities` / `agent-version` always emit + pure JSON to stdout regardless of TTY** — no Rich coloring, no + `--json` flag needed, no banner. They're machine endpoints first. + Tests assert generated schema shape and CLI emission across patches. +- **Snapshot regeneration**: changing `find --json` from a string array + to a `FileSearchResult` object invalidates + `tests/__snapshots__/test_cli.ambr::test_find_default_output` and + related snapshots. Phase 0d regenerates them via + `pytest --snapshot-update` and commits the diff alongside the schema + flip. Reviewers should diff the snapshots against the new schema to + confirm shape parity. + +**Critical files**: `src/rglob/agent/_models.py` (new), +`src/rglob/agent/_introspection.py` (runtime schema generation), +`src/rglob/cli.py`, +`tests/fixtures/agent-tree/` (new), `tests/test_agent_contract.py` (new), +`tests/__snapshots__/test_cli.ambr` (regenerated for the new +`find --json` shape), +`docs/decisions/0009-agent-api-contract.md` (new), +`docs/decisions/0010-agent-safety-model.md` (new). + +**Acceptance**: `rglob schema find | jq` validates as JSON Schema +Draft 2020-12; `rglob capabilities --json` reports schema/API versions +and installed extras; golden fixture snapshots prove Python API, CLI +JSON, and future MCP output use the same record shapes; invalid regexes, +unsupported predicates, unreadable paths, and truncated searches all +produce stable machine-readable errors or warnings. + +--- + +## Phase 1 — Scope expansion (`grep` + `find` predicates) + +Land the missing `find(1)` / `grep` capabilities so the CLI is genuinely +"all your search needs", not just "globbing with filters". + +**Tasks** + +- New module `src/rglob/_grep.py`: regex / fixed-string content matcher + with `before`/`after`/`context` line capture. Iterates files yielded + from `find()`. Uses `re` (stdlib) — no `regex` extra in 2.0; revisit + if PCRE features are demanded. The matcher returns Phase 0 + `LineMatch` records, not ad hoc dicts. +- Grep defaults are agent-safe and explicit: UTF-8 with + `errors="replace"`, binary files skipped by default, `--text/-a` to + force binary-as-text, `--encoding` for override, `--max-file-size` and + `--max-bytes` for output control, and stable regex-compile errors. +- New CLI subcommand `rglob grep [files-or-globs...]` with + flags: `--fixed-string/-F`, `--ignore-case/-i`, `--line-number/-n`, + `--context/-C N`, `--before/-B N`, `--after/-A N`, `--max-count/-m N`, + `--word/-w`, `--invert/-v`, `--encoding`, `--text/-a`, `--limit`, + `--max-bytes`, `--max-file-size`. Honours all the existing `find` + filter flags (`--exclude`, `--max-depth`, `--hidden`, `--type`, + `--gitignore`, `--min-size`, `--newer-than`, etc.). +- New CLI subcommand `rglob count ` (split out from today's + `lcount`/`tsize` so the verb is consistent: `count` reports files + + lines + bytes in one structured view). `lcount`/`tsize` stay as + aliases for backward-compatibility. +- Extend `find()` with the `find(1)`-style filters we don't have: + `mtime`/`ctime`/`atime` ranges (already partially via `newer_than` / + `older_than`), `perm`-mask matching, `uid`/`gid` matching (POSIX-only; + reported as unsupported on Windows rather than silently no-oping), + `newer_than_file=Path` shortcut (mirroring `find -newer FILE`). + This work includes: + - New predicate helpers in `src/rglob/_filters.py`. + - Updates to `WalkOptions` and the core walker logic in + `src/rglob/rglob.py` (must respect `strict_base` and feed into + the error envelope / `ErrorInfo`). + - Corresponding entries in `CapabilityReport.predicates` + (`"supported"`, `"POSIX-only"`, or `"unsupported"`). + - Cross-platform tests and clear documentation of Windows behaviour. +- Output formats unified: `--json` and `--jsonl` work on `find`, `grep`, + `count`. The schemas are version-locked (Phase 0) and emitted via + `rglob schema`; legacy `find --json` string arrays may remain under + `--json-paths` only if compatibility demands it. +- Revisit `.gitignore` semantics before agents depend on it. Document + the current `pathspec` approximation honestly, add tests for nested + `.gitignore` files, and decide whether Phase 1 needs a small + per-directory matcher improvement. + +**Critical files**: `src/rglob/_grep.py` (new), `src/rglob/cli.py` (new +subcommands), `src/rglob/rglob.py` (predicate extensions), +`tests/test_grep.py` (new), `docs/cli/grep.md` (new). + +**Acceptance**: `rglob grep TODO src/ --json | jq '.results[] | .path'` +returns the expected matches; `rglob count "*.py" --jsonl` round-trips +through `jq`; binary files, bad regexes, invalid encodings, context +groups, and truncation are covered; **aggregated coverage stays at +100%**; no hard new dependencies introduced; CHANGELOG `[2.0.0]` +accumulates the new surface under a "Phase 1 — Agent platform: scope +expansion" subsection. + +--- + +## Phase 2 — Agent API contract (`rglob.agent`) + +The implementation phase for the Phase 0 contract. Expose the +agent-facing Python namespace so 2.x is a contract agents can integrate +against without fear of churn. + +**Tasks** + +- New top-level module `src/rglob/agent/__init__.py` re-exporting *only* + the agent-stable surface. Initial members: + - Dataclasses (frozen, slotted; defined in Phase 0): `FileMatch`, + `LineMatch`, `Stats`, `Duplicate`, `ErrorInfo`, `ErrorCode`, + `FileSearchResult`, `LineSearchResult`, `DuplicateSearchResult`, + `CapabilityReport`, `WalkOptions`, `GrepOptions`, `CountOptions`. + - Functions (typed end-to-end): + `search(opts: WalkOptions) -> Iterator[FileMatch]`, + `search_all(opts: WalkOptions) -> FileSearchResult`, + `grep(opts: GrepOptions) -> Iterator[LineMatch]`, + `grep_all(opts: GrepOptions) -> LineSearchResult`, + `count(opts: CountOptions) -> Stats`, + `find_duplicates(opts: WalkOptions) -> DuplicateSearchResult`. + - Error handling contract (locked for the agent API): + The functions `search`, `search_all`, `grep`, `grep_all`, `count`, + and `find_duplicates` are **non-raising** for operational errors + (permission denied, timeout, unreadable files, binary files, + bad regex, unsupported predicates, etc.). All such conditions + are reported via the `errors: list[ErrorInfo]` field on the + returned `FileSearchResult` / `LineSearchResult` / `DuplicateSearchResult` + (or on `Stats` for `count`). + Only genuine programmer errors may raise. This contract must be + documented in the function docstrings and in + `docs/agents/python-api.md`. + - Constants: `__agent_api_version__ = "1.0"` (SemVer-locked + independently of package `__version__` so the CLI/library can + evolve faster than the agent contract). +- JSON schemas (JSON Schema Draft 2020-12) come from the Phase 0 runtime + generator in `src/rglob/agent/_introspection.py`. No runtime dep on + `pydantic`; schemas are emitted by `rglob.agent.schema_for()`, + `rglob.agent.all_schemas()`, `rglob schema `, and + `rglob schema --all`. +- New CLI introspection: `rglob describe ` prints a JSON + manifest of arguments + types + defaults; `rglob schema ` + prints the input + output JSON schemas; `rglob agent-version` prints + `__agent_api_version__`. Tests assert these endpoints' outputs are + byte-stable across patch releases. +- The agent functions share implementation with CLI JSON. They should + not re-wrap private helpers in a second, drifting way; the golden + fixture snapshots from Phase 0 are the guardrail. +- Stability rules documented in + `docs/decisions/0009-agent-api-contract.md` (written in Phase 0a): + - Adding a field to a dataclass / a flag to a CLI command = minor + bump on `__agent_api_version__`. + - Removing or renaming = major bump. + - JSON schemas have an `$id` and `version` field; the contract is + that **`rglob` will always emit valid v1.x schemas while + `__agent_api_version__` starts with `"1."`**. + - The `rglob.agent` import path is the only path agents should pin + against. Importing from `rglob` directly continues to work but is + "best-effort" stable. + +**Critical files**: `src/rglob/agent/__init__.py` (new), +`tests/test_agent_api.py` (new). (`_models.py`, `_introspection.py`, the +ADRs, and `tests/test_agent_contract.py` are owned by Phase 0; this phase +only adds the public re-exports and consumer-facing tests.) + +**Acceptance**: `from rglob.agent import search, FileMatch` works on a +fresh `pip install -e .`; `rglob schema find | jq '.input.required'` +returns the locked input schema; `mypy --strict` passes against the +agent module from a downstream consumer; one full round-trip +documented in `docs/agents/python-api.md`. + +--- + +## Phase 3 — MCP server (`rglob mcp`) + +Ship `rglob` as a stdio MCP server so Claude Code / Cursor / any +MCP-aware host can discover and call it natively, no shell-out +required. + +**Tasks** + +- New optional extra: `pip install rglob[mcp]` pulls in the official + `mcp` Python SDK **plus** `pathspec` and `xxhash` so `--gitignore` and + `dupes` work out of the box (`mcp = ["mcp>=1.0", "pathspec>=0.12", + "xxhash>=3.4"]`). +- Before coding `src/rglob/agent/mcp.py`, verify the installed official + `mcp` SDK API in a throwaway import smoke test and update the rough + sketch below if the actual `Server`, `@tool`, or stdio runner names + differ. The plan's contract is the tool list and JSON shape, not the + exact SDK spelling in the sketch. +- New entry point: `rglob mcp` (subcommand) launches a stdio MCP server + exposing tools: + - `find_files(pattern, **filters) -> FileSearchResult` + - `grep_content(pattern, paths, **opts) -> LineSearchResult` + - `count_lines(pattern, **filters) -> Stats` + - `find_duplicates(pattern, **filters) -> DuplicateSearchResult` + - `describe_subcommand(name) -> dict` (mirrors `rglob describe `) + + Rough implementation shape (in `src/rglob/agent/mcp.py`): + + ```python + import asyncio + from pathlib import Path + + from mcp.server import Server + + from rglob.agent import WalkOptions, search_all, to_json_dict + + mcp = Server("rglob") + + + @mcp.tool() + async def find_files(pattern: str, base: str = ".", **filters) -> dict: + opts = WalkOptions( + base=Path(base), + patterns=[pattern], + strict_base=True, # conservative MCP defaults + follow_symlinks=False, + limit=5000, + **filters, + ) + return to_json_dict(search_all(opts)) + + # similar @mcp.tool() for grep_content, count_lines, find_duplicates, etc. + + def main() -> None: + asyncio.run(mcp.run_stdio_async()) + ``` + + The Typer `mcp` command simply calls this `main()` after optional + banner / version checks. All tools return the same concrete result + shapes (`FileSearchResult` / `LineSearchResult` / + `DuplicateSearchResult`) used by the CLI and Python API; the + shared `to_json_dict()` helper gives the MCP layer JSON-safe `Path` / + `datetime` / `StrEnum` serialisation without dragging in `pydantic`. + +- MCP tools keep the Phase 0 conservative defaults: bounded result + counts, bounded bytes, `strict_base=True`, `follow_symlinks=False`, + and explicit truncation metadata. Agents can opt into larger searches, + but the server must never dump an unbounded repository over stdio by + default. +- Each tool's input/output schema is **derived from the same Phase 0/B + dataclasses** — single source of truth, no schema drift between CLI + JSON output and MCP tool I/O. +- Sample MCP client config snippets for Claude Code, Cursor, generic + hosts in `docs/agents/mcp-setup.md` (new doc subdirectory). +- Integration test: spin up the server in a subprocess, send tool calls + via the `mcp` SDK's stdio transport, verify the responses match the + equivalent CLI JSON records from the golden fixture. Include truncated + results and error-envelope cases. + +**Critical files**: `src/rglob/agent/mcp.py` (new server module), +`tests/test_mcp.py` (new), `docs/agents/mcp-setup.md` (new), +`pyproject.toml` (the `mcp` extra). + +**Acceptance**: `pip install -e .[mcp] && rglob mcp` starts and +responds to a `tools/list` request listing all five tools; Claude Code +with the documented config snippet successfully calls `find_files` and +gets typed, bounded results; coverage of `agent/mcp.py` stays at 100% +(mocked transport in tests); the MCP server is genuinely useful — +verified by asking Claude to find duplicate files and watching it +succeed without hand-holding or runaway output. + +--- + +## Phase 4 — Documentation push (agent-first) + +Today's docs assume a human reader. The agent platform additions make +agent-facing docs first-class. + +**Tasks** + +- Split `AGENTS.md` into two sections (or two files): + 1. **Contributor guidance** (today's content — kept). + 2. **Consumer guidance** (new) — for agents *using* `rglob` in their + projects. Includes: how to install, how to discover via + `rglob describe` / `rglob schema`, the JSON output schemas, MCP + setup, the stability promise. +- New `docs/agents/` subdirectory: + - `index.md` — the consumer landing page. + - `mcp-setup.md` — copy-paste configs for Claude Code / Cursor / + generic MCP hosts. + - `python-api.md` — typed Python integration guide with + `from rglob.agent import ...` examples. + - `cli-recipes.md` — common subprocess patterns with `subprocess.run` + and friends. + - `stability.md` — the SemVer + agent-API contract spelled out. + - `safety.md` — safe defaults, path containment, output limits, + binary grep behaviour, and what file contents may be exposed. +- New `docs/examples/` directory — every recipe is *runnable* via the + Phase 0-locked custom harness (`tests/examples_harness.py`); both + bash and Python blocks execute in CI so examples cannot rot. + Initial set: + - "Find all Python files modified in last 7 days, as JSON." + - "Grep TODOs across `src/`, excluding `.venv` and `dist`, with + surrounding context." + - "Count non-blank, non-comment lines per language." + - "Detect duplicate files in `~/Downloads` larger than 1 MiB." +- Self-describing CLI: every subcommand wired to the `rglob describe` / + `rglob schema` introspection from Phase 0/B; surfaced in + `cli-recipes.md`. +- README rewrite of the top fold to lead with the agent-friendly + positioning: ASCII banner → tagline → "**Designed to be the default + recursive-search dependency for coding agents**" → install → MCP / + Python / CLI examples in three columns / tabs. +- README accuracy pass: current 2.0 code returns `list[Path]` from + `rglob()` / `rglob_()`; remove any remaining `list[str]` wording from + README and docs before agent-facing examples land. +- `mkdocs.yml` nav update for `docs/agents/`, `docs/examples/`, and ADRs + 0009/0010. `docs/changelog.md` must still include the root + `CHANGELOG.md` without duplicate stale release text. + +**Critical files**: `AGENTS.md` (consumer half added), `docs/agents/*` +(new tree), `docs/examples/*` (new tree), `README.md` (top-fold +rewrite), `mkdocs.yml`, `docs/changelog.md`, `SECURITY.md`, +`tests/examples_harness.py` (runs the example recipes via the +Phase 0-locked custom harness). + +**Acceptance**: a brand-new agent (test: spawn a fresh Claude Code +session, point it at the README, ask it to "use rglob to find +duplicate PNGs over 1MB in this dir" — measure success on first try); +every doc example runs green in CI; `rglob describe` / `rglob schema` +outputs are snapshot-tested via syrupy. + +--- + +## Phase 5 — Final 2.0 polish before tag + +The single PyPI release point for 2.0 still applies — these phases +land on the same PR and ship together. + +**Tasks** + +- `__version__` stays `"2.0.0"`. Confirm `Development Status` classifier + remains `6 - Mature` (decided in the original Phase 6 of the + modernization roadmap). +- `CHANGELOG.md` `[2.0.0]` section absorbs the agent-platform additions + under clearly-titled "Phase 0 / 1 / 2 / 3 / 4 — Agent platform" + subsections; release date is refreshed when the tag is pushed. +- Architecture diagram refresh: add Mermaid `classDiagram` of the + `rglob.agent` namespace, sequence diagram of an MCP request + lifecycle. +- Release packaging check: generated schemas work from an installed wheel + without packaged `_schemas/*.json`; `py.typed` and any `docs/examples/` + test data are present in the wheel/sdist where expected; `rglob schema + find` works from an installed wheel, not only from an editable checkout. +- README final pass — the top-fold rewrite from Phase 4 becomes the + canonical 2.0 README. +- Tag `v2.0.0` → `release.yml` workflow does PyPI OIDC publish + GH + Release. + +**Critical files**: `src/rglob/__init__.py` (version unchanged at +`2.0.0`), `src/rglob/agent/__init__.py`, `pyproject.toml`, +`CHANGELOG.md`, `docs/architecture.md`, `README.md`, +`.github/workflows/ci.yml`, `.github/workflows/release.yml`. + +**Acceptance**: `pip install rglob==2.0.0 && rglob mcp --help` works; +`from rglob.agent import search, FileMatch` works in a fresh venv; +`rglob.agent.__agent_api_version__ == "1.0"`; aggregated coverage = +100%; the docs site renders the agent landing page; one full +Claude-Code-driven "find duplicate downloads" task succeeds on a fresh +install of 2.0 from PyPI. + +--- + +## Critical files (cross-phase quick reference) + +- `src/rglob/agent/` — new top-level subpackage holding the locked + contract (Phase 0 defines the records/schemas; Phase 2 exposes the + public Python API; Phase 3 extends it; Phase 4 documents it). +- `src/rglob/_grep.py` — new content-search engine (Phase 1). +- `src/rglob/cli.py` — gains `grep` / `count` / `mcp` subcommands and + `describe` / `schema` introspection (Phases 1–3). +- `pyproject.toml` — new `mcp` extra (Phase 3). +- `tests/fixtures/agent-tree/` — shared golden fixture proving CLI, + Python API, and MCP records stay aligned. +- `docs/agents/` and `docs/examples/` — new doc subtrees (Phase 4). +- `AGENTS.md` — consumer-facing half added (Phase 4). +- `README.md` — top fold rewritten to lead with the agent positioning + (Phase 4, refined in Phase 5). +- `docs/decisions/0009-agent-api-contract.md` — the SemVer + stability + ADR (Phase 0/2). +- `docs/decisions/0010-agent-safety-model.md` — path containment, + content disclosure, and output-limit ADR (Phase 0). +- `mkdocs.yml`, `.github/workflows/ci.yml`, `.github/workflows/release.yml` + — docs navigation, CI extras, and release packaging must reflect the + new schema/MCP surface. + +--- + +## Verification + +End-to-end checks per phase. As with the modernization roadmap, no +PyPI publishes happen until the single 2.0 tag. + +- **Phase 0**: `rglob schema find | jq` validates as JSON Schema + Draft 2020-12; `rglob capabilities --json` reports extras and + platform-supported predicates; golden fixture snapshots cover success, + errors, and truncation. +- **Phase 1**: `rglob grep TODO src/ --json | jq` round-trips; `rglob + count "*.py" --jsonl` works; full coverage maintained. +- **Phase 2**: `python -c "from rglob.agent import search, FileMatch"` + succeeds; downstream `mypy --strict` accepts the exported types; + ADR-0009 and ADR-0010 are reflected in the public docs. +- **Phase 3**: `pip install -e .[mcp]` + `rglob mcp` accepts a + `tools/list` MCP request; integration test against a mocked stdio + transport passes; documented Claude Code config produces a working + tool call. +- **Phase 4**: `[ ]` every `docs/examples/*.md` runs as part of + `make test` after `tests/examples_harness.py` lands; `[ ]` + `make docs-build` strict-mode clean after the final docs pass; `[x]` + new agent landing page content exists in the working tree. +- **Phase 5**: `[ ]` after tag/publish, `pip install rglob==2.0.0` from + PyPI gives an importable `rglob.agent`, a working `rglob mcp` server, + and a green smoke test driven by a real coding agent. + +--- + +## Open questions to resolve before Phase 0 / Phase 1 starts + +Some questions (CLI introspection syntax, schema generation approach, +golden fixture layout, `mcp` extra composition, and phase numbering) +were locked during the pre-implementation pass and are documented in +Phase 0 and Phase 3. The remaining open questions below will be answered +during the Phase 0 design pass. + +1. **`grep` regex flavour** — stick with stdlib `re` (POSIX-ish, + Python-flavoured), or expose `regex` as an optional extra for + look-around / Unicode-class power users? Default proposal: stdlib + `re` in 2.0; revisit in 2.x if asked. +2. **`grep` on binaries** — skip-by-default like ripgrep, with + `--text/-a` to force? Default proposal: yes. +3. **MCP transport** — stdio is non-negotiable for Claude Code / + Cursor. Should we *also* ship SSE/HTTP variants? Default proposal: + no in 2.0. +4. **Versioning of `__agent_api_version__`** — separate SemVer line + from package version, or always lockstep? Default proposal: + separate, so the CLI / library can patch without touching the agent + contract. +5. **`docs/examples/` runner** — *Locked*: a small custom harness in + `tests/examples_harness.py` that walks `docs/examples/*.md`, + extracts fenced \`\`\`bash and \`\`\`python blocks tagged with a + magic header line (``), + executes bash blocks via `subprocess.run` against a freshly-built + wheel and Python blocks via `exec` in a per-example namespace, and + compares stdout to a sibling `` block when + present. Doctest is too narrow (no shell coverage) and + `pytest-codeblocks` adds a runtime dep we don't need. +6. **Default output limits** — what are the MCP and structured-CLI + defaults for `limit`, `max_bytes`, and `timeout_seconds`? Default + proposal: MCP is bounded by default; CLI path streaming is unbounded; + CLI `--json` / `--jsonl` reports truncation and accepts explicit + limit flags. +7. **Strict base containment** — should `strict_base=True` become a + public `find()` default or only the agent/MCP default? Default + proposal: keep the existing public walker behaviour for compatibility, + but make agent/MCP tools strict by default. +8. **Structured JSON compatibility** — should current `find --json` + remain a string array, or can it become a `SearchResult` object before + 2.0 ships? Default proposal: use the better `SearchResult` shape now + because 2.0 is not released yet; offer `--json-paths` only if needed. + +These are answered in-line during Phase 0's design pass; the defaults +above are what we'll execute unless flagged otherwise. diff --git a/docs/plans/modernization-roadmap.md b/docs/plans/modernization-roadmap.md new file mode 100644 index 0000000..03a3808 --- /dev/null +++ b/docs/plans/modernization-roadmap.md @@ -0,0 +1,223 @@ +# `python-rglob` Modernization & Feature Expansion + +## Context + +`python-rglob` is a ~120-LOC hobby package Chris built ~5–6 years ago when Python 2 had no decent recursive-glob story. It was upgraded to Python 3 but otherwise hasn't evolved: hand-rolled recursion via `glob.iglob`, no `pathlib`, lists not generators, no type generics, no symlink/hidden/`.gitignore` handling, argparse CLI, pylint-only CI, behave BDD tests with no pytest, `setup.py`-driven metadata, committed `egg-info`, and a `1.7` source / `v1.8` tag drift. Modern alternatives exist (`pathlib.Path.rglob`, `fd`, `wcmatch`), and that's fine — this repo is a labor of love, and the redundancy is the point. + +The goal is to (a) drag the project up to current best practices and (b) borrow the popular features that make `fd`/`ripgrep`/`wcmatch` fun to use, while keeping each phase small enough to enjoy on a weekend. Each phase ships independently and leaves `main` releasable. + +**Decisions locked in**: Python **3.11 floor** (3.10 reaches EOL in October 2026 — bumping ahead of EOL keeps us forward-looking); `rglob()` returns `list[Path]` at 2.0; Typer + Rich for the CLI; full six-phase scope; **single version bump to `2.0.0` after all six phases land** (no intermediate PyPI releases). During development the source version stays at PEP 440 `"2.0.0.dev0"` so installs are clearly marked as work-in-progress. + +**Sensible defaults applied without asking**: Ruff replaces pylint; skip async; keep `behave` as a parallel BDD job (it's charming, with the understood maintenance cost that scenarios will rot first — call out in a decisions ADR); `hatchling` build backend; `src/` layout; MkDocs Material; Renovate over Dependabot; Codecov for coverage (aggregated across the matrix — see Phase 2 coverage notes); OIDC trusted publishing to PyPI. + +--- + +## Phase 1 — Packaging hygiene + +Pure plumbing. Zero behaviour change. Clears the runway for everything else. + +**Tasks** +- Migrate metadata from `setup.py` to PEP 621 `[project]` in `pyproject.toml`; build backend = `hatchling`. Delete `setup.py`. +- Single-source `__version__ = "2.0.0.dev0"` in `src/rglob/__init__.py`; `[tool.hatch.version] path = "src/rglob/__init__.py"`. Closes the 1.7/v1.8 drift and signals WIP until Phase 6. +- Bump `requires-python = ">=3.11"`; refresh trove classifiers (drop 3.5, add 3.11–3.14, add `Topic :: System :: Filesystems`, `Environment :: Console`, `Typing :: Typed`). +- Repo layout: `git mv rglob/ src/rglob/`. Update import paths in `features/steps/steps.py` (no change needed — it imports the installed package). +- `git rm -r --cached rglob.egg-info/`. Update `.gitignore` to also list `dist/`, `build/`, `.venv/`, `.pytest_cache/`, `.mypy_cache/`, `.ruff_cache/`, `.coverage*`, `htmlcov/`. +- Add `CHANGELOG.md` (Keep a Changelog format with a single `[Unreleased]` section that accretes through Phases 1–5), `CONTRIBUTING.md` (install/test/lint; reference `src/rglob` paths only — no `setup.py` left to mention), `SECURITY.md` (one-line: GitHub private advisories, expanded with the symlink/path-traversal threat model in Phase 3), `CODE_OF_CONDUCT.md` (Contributor Covenant 2.1 drop-in). +- README overhaul (the README is a first-class artifact, kept accurate through every phase): + - **Top-level ASCII banner** at the very top — a `figlet`/`pyfiglet`-style "rglob" banner in a fenced code block (font: `slant` or `ANSI Shadow`; pick whichever reads best in a GitHub-rendered monospace block). + - Tagline + badges row directly under the banner: PyPI / Python versions / CI / Codecov / license / ruff / "code style: ruff". + - Accuracy pass: drop or rewrite anything that's wrong post-modernization (today's README claims `**` isn't supported and "paths are not guaranteed to be sorted" — both flip in Phase 3; update with each phase that touches the API). + - Sections: Why use this? (lean into "modern alternatives exist — that's part of the fun"), Install, Quick Start (Python API), Quick Start (CLI), Compatibility, Links to full docs (`docs/`), Development, License. + - Add a "Documentation" section linking to the MkDocs site (set up in Phase 2) and to `docs/architecture.md`. + +**Critical files**: `pyproject.toml`, `src/rglob/__init__.py`, `.gitignore`, `CHANGELOG.md`, `README.md`. + +**Acceptance**: `pip install -e .` works; `python -c "import rglob; print(rglob.__version__)"` prints `2.0.0.dev0`; existing pylint CI still passes (we replace it in Phase 2). No tag, no PyPI release. + +--- + +## Phase 2 — Tooling overhaul + +Build the dev loop you'll want to live in for the rest of the phases. `release.yml` is wired up here but no tag is cut until Phase 6. + +**Tasks** +- Replace pylint with **ruff** (lint + format) in `pyproject.toml`: + - `[tool.ruff.lint] select = ["E","F","W","I","B","UP","SIM","RUF","PTH","PERF","D"]`, pydocstyle convention `"google"`. + - Delete the pylint `[tool.pylint.*]` sections; uninstall pylint. +- Add **mypy** with `strict = true` on `src/rglob`; ship a `src/rglob/py.typed` marker. +- Add `.pre-commit-config.yaml`: ruff, mypy, check-yaml, check-toml, end-of-file-fixer, trailing-whitespace, pyproject-fmt. +- Add **pytest** in `tests/`: port the behave step logic into pytest functions using `tmp_path` and `monkeypatch`. Keep `features/` as a parallel BDD suite; have its steps call the same helpers `tests/` use. +- Coverage via `coverage[toml]` configured in `pyproject.toml` (delete `.coveragerc` — Phase 1 hangover). **Target 100% on the *aggregated* report (Codecov merges all matrix jobs).** Per-job `fail_under` is set to **95%** so any single-OS regression is loud but doesn't false-alarm on platform-conditional branches; the *merged* report is what must be 100% green (enforced via the Codecov `target: 100%` project gate). Use `# pragma: no cover` only for genuinely un-coverable lines (e.g., `if __name__ == "__main__":` guards, `if sys.platform == "win32"` blocks that the corresponding OS job *does* cover but Linux runs can't see). Every pragma must have a one-line justification comment. Codecov upload + badge. +- Replace `.github/workflows/pylint.yml` with three workflows: + - `ci.yml`: matrix `python-version: [3.11, 3.12, 3.13, 3.14]` × `os: [ubuntu, macos, windows]`. Steps: `uv sync --all-extras --dev` (or fallback `uv pip install -e .[dev,bdd,docs,gitignore]`), `ruff check`, `ruff format --check`, `mypy --strict src/rglob`, `pytest --cov --cov-fail-under=95`, `behave`, upload coverage to Codecov. **All steps are gating** — a failure on any of lint / type / test / per-job-coverage blocks merge. The Codecov *project* status check enforces the 100% aggregated bar. + - `release.yml` (on `v*` tag): `hatch build` → PyPI via OIDC trusted publishing → GitHub Release from CHANGELOG slice. + - `docs.yml` (on push to main): MkDocs Material build + deploy to `gh-pages`. +- **Enforce lint + tests via GitHub branch protection on `master`**: required status checks include all matrix `ci / test (*)` jobs and `ci / lint`; require PRs (no direct push); require linear history; require up-to-date branches before merging. One-time setup in repo settings → Branches (document the exact settings in `CONTRIBUTING.md`). +- `pre-commit install` is the local first line of defence — ruff, mypy, and a `pytest -x --no-cov` quick run on staged-file-related tests so lint/test failures get caught before push. +- Add `renovate.json` (preset `config:base`, schedule weekly). +- Add MkDocs Material skeleton with **Mermaid support** enabled: + - `mkdocs.yml` — theme `material`, navigation, and `markdown_extensions: [pymdownx.superfences]` with a `mermaid` custom fence so ` ```mermaid ` blocks render natively. + - `docs/index.md` — landing page mirroring README (without the duplicated banner). + - `docs/api.md` — auto-generated from docstrings via `mkdocstrings[python]`. + - `docs/cli.md` — placeholder; gets real content via `mkdocs-typer2` (the actively-maintained Typer doc plugin; the older `mkdocs-click` bridge required `typer.main.get_command()` and is now redundant) once CLI migrates in Phase 4. + - `docs/architecture.md` — design overview seeded with Mermaid diagrams in this phase, expanded in each later phase (see "Architecture diagrams" below). + - `docs/decisions/` — lightweight ADR-style notes capturing the decisions locked in this plan (build backend, Python 3.11 floor, Path return at 2.0, Typer + Rich, single-release strategy, aggregated 100% coverage target, behave-as-parallel-suite trade-off). + - `docs/contributing.md` — includes `CONTRIBUTING.md` via `mkdocs` snippet plugin (single source of truth). + - `docs/plans/` — this plan lives here; later plans can land alongside it. +- **Architecture diagrams in `docs/architecture.md`** (added incrementally per phase; all in Mermaid so they render on GitHub and the docs site): + - **Phase 2**: package layout (`graph TD` of `src/rglob/`, `tests/`, `features/`, `docs/`, `.github/workflows/`) showing module relationships and CI flow. + - **Phase 3**: walker call-graph (`flowchart` of `find` → scandir → match → filter → yield) and symlink-loop detection sequence diagram. + - **Phase 4**: CLI command hierarchy (`graph TD` of `rglob` → subcommands → shared filter options); rendered Rich output layout. + - **Phase 5**: `dupes` pipeline (`flowchart LR`: size bucket → 4 KB hash bucket → full hash bucket); `.gitignore` pruning sequence. + - **Phase 6**: public API surface (Mermaid `classDiagram`) showing the final 2.0 shape. +- Rewrite `Makefile` with the user-required quartet plus a few essentials. Default goal is `help` (typing bare `make` prints the menu): + - `make help` — auto-discovered list of targets with one-line descriptions (using `## ` doc-comment convention parsed by an awk one-liner). + - `make build` — `hatch build` produces sdist + wheel into `dist/`. + - `make lint` — `ruff check .` and `ruff format --check .` and `mypy --strict src/rglob` (gating; non-zero exit on any failure). + - `make test` — `pytest --cov --cov-fail-under=100` and `behave` (gating; the local run sees a full single-OS profile, so 100% is enforced locally — CI relaxes per-job to 95% because the matrix splits coverage across OSes). + - `make fmt` — `ruff format .` and `ruff check --fix .`. + - `make docs` — `mkdocs serve` (live preview on `:8000`); `make docs-build` for a static build. + - `make dev-setup` — installs `.[dev,bdd,docs,gitignore]` and runs `pre-commit install`. + - `make clean` — removes `dist/`, `build/`, `.pytest_cache/`, `.mypy_cache/`, `.ruff_cache/`, `htmlcov/`, `.coverage*`, `site/` (mkdocs). +- Refresh `requirements-dev.txt` → `[project.optional-dependencies]` in `pyproject.toml`: `dev`, `bdd`, `docs`, `gitignore` (Phase 5), `ext` (Phase 6 stretch) as separate extras. + +**Critical files**: `pyproject.toml`, `.pre-commit-config.yaml`, `.github/workflows/{ci,release,docs}.yml`, `tests/test_*.py` (new), `mkdocs.yml`, `docs/architecture.md` (new, with Mermaid), `docs/index.md`, `docs/api.md`, `docs/cli.md`, `docs/contributing.md`, `docs/decisions/*.md`, `Makefile`, `README.md`. + +**Acceptance**: `make help` lists every target with descriptions; `make lint test build docs` all green locally with **100% local coverage**; CI matrix green on PRs; **aggregated coverage = 100%** on Codecov merged report with every `# pragma: no cover` justified; branch protection on `master` blocks a PR with a failing test or lint; docs site at `https://chris-piekarski.github.io/python-rglob/` renders Mermaid diagrams correctly (verify by viewing `architecture.md`); one round-trip dry-run of `release.yml` against **TestPyPI** confirms OIDC config (no tag pushed to real PyPI yet). + +--- + +## Phase 3 — Core API additions (additive only) + +Rewrite internals on `os.scandir`; keep `rglob(base, pattern) -> list[str]` byte-compatible with today's signature; introduce the new `find()` family. Tier 1 features only. All changes accrete into the `[Unreleased]` section of CHANGELOG. + +**Tasks** +- Rewrite the walker in `src/rglob/rglob.py` on `os.scandir` with recursion. `os.scandir` returns `is_dir()`/`is_symlink()` from cached `dirent.d_type` — drops a `stat()` per entry. Drop `_get_dirs` + `glob.iglob`. +- Keep existing functions working unchanged: `rglob(base, pattern) -> list[str]`, `rglob_(pattern) -> list[str]`, `lcount`, `tsize`, `kilobytes`/`megabytes`/`gigabytes`/`terabytes`. These become thin wrappers that coerce `Path → str` and `list(generator)`. +- New canonical API in `src/rglob/rglob.py`: + - `find(base, patterns, *, exclude=(), max_depth=None, hidden=False, follow_symlinks=False, case_sensitive=None, sort=True, on_error="warn") -> Iterator[Path]` + - `find_all(...) -> list[Path]` (just `list(find(...))`) + - Accept `str | os.PathLike[str]` for `base`; accept `str | Sequence[str]` for `patterns`. +- `**` recursive glob support: per-component `fnmatch` matching instead of relying on `glob.glob`. Today's README disclaims `**` — flip that. +- Symlink loop detection via a visited-`os.path.realpath` memo. Realpath (not `Path.resolve(strict=True)`) so we don't choke on dangling links; symlink loops terminate; the memo is scoped per `find()` call to avoid cross-call leaks. Add a security ADR in `docs/decisions/` describing the threat model: in-scope = symlink-escape and traversal *outside* the user-supplied `base`; out-of-scope = TOCTOU between `scandir` and `lstat`; mitigations = `follow_symlinks=False` default, realpath memo, `respect_gitignore` never reads files outside `base`. Add a unit test that builds a symlink cycle and asserts termination. +- Deterministic ordering: `sort=True` by default (current behaviour is undocumented "whatever scandir returns" — flip to documented "sorted by path"). README today warns "not guaranteed sorted"; we promise sorted, opt-out via `sort=False`. +- `case_sensitive=None` follows the OS (case-insensitive on Windows/macOS by default), matching `pathlib.PurePath.match` in 3.12+. +- Error handling: `on_error: Literal["ignore","warn","raise"] = "warn"`. Catches `PermissionError`, `OSError`, `UnicodeDecodeError` on entry names. Today's code would explode on `/proc` or a permission-denied dir. +- Modern type hints: PEP 604 unions, `collections.abc` for `Iterable`/`Iterator`/`Sequence`. Drop `from __future__ import annotations` — with the 3.11 floor, PEP 604 unions and built-in generics work natively at runtime, so the future-import is just dead weight. (Typer-compat concerns from older versions no longer apply; Typer ≥0.9 handles stringified annotations correctly.) +- Update `src/rglob/__init__.py` to export `find`, `find_all` alongside the legacy names. +- Hypothesis property tests in `tests/test_properties.py`: + - `find(p, "*")` cardinality matches a brute-force `os.walk` reference. + - Monotonicity in `max_depth`. + - Exclude-pattern commutativity. + - `case_sensitive=False` set ⊇ `case_sensitive=True` set. +- Docs: new "Modern API" page; mark old API as "legacy (still supported)"; expand `docs/architecture.md` with the walker call-graph + symlink-loop sequence diagrams in Mermaid. +- README: update Quick Start examples to feature `find()`/`find_all()` alongside the legacy `rglob()`; remove the "`**` is not supported" disclaimer; flip the "paths not guaranteed sorted" note to "paths sorted by default; pass `sort=False` for raw scandir order". + +**Critical files**: `src/rglob/rglob.py`, `src/rglob/__init__.py`, `tests/test_find.py` (new), `tests/test_properties.py` (new), `docs/api.md`, `docs/architecture.md`, `README.md`. + +**Acceptance**: every existing behave scenario still passes unchanged; new pytest+hypothesis tests cover the new flags; **aggregated coverage stays at 100%** (new code is fully exercised); `find()` benchmark on a 10k-file synthetic tree (10 dirs × 1000 files, mean depth 3, **warm FS cache**, 5 runs, comparing `pytest-benchmark` *median*) is **≥2× faster** than the legacy `rglob()` on Linux ext4/tmpfs (scandir win — the relevant scenario for everyday use); `mypy --strict` clean; README + architecture diagrams reflect the new API. + +--- + +## Phase 4 — CLI overhaul + +Migrate argparse → **Typer + Rich**. Keep existing subcommand surface byte-compatible; layer on all the new Phase 3 filters. + +**Tasks** +- Rewrite `src/rglob/cli.py` on Typer. Keep `find`, `lcount`, `tsize` subcommands and their existing flags (`--base`, `--no-empty`, `--no-comments`, `--unit`). +- Add filter flags mirroring the core: `--exclude/-E` (multi), `--max-depth/-d`, `--hidden/-H`, `--follow/-L`, `--type/-t {f,d,l,x}`, `--size +1M -10M` (fd-style range), `--newer-than`/`--older-than` (accept `7d`, `2024-01-01`, etc.), `--gitignore/--no-gitignore` (stub for Phase 5), `--case-sensitive/-s`. +- Output formats: default one-path-per-line (unchanged); `--json` / `--jsonl`; `-0`/`--null` for `xargs -0` safety; `--format` mini-template (`"{path} {size:human}"`). +- Rich integration: colored help, colored output, auto-progress bar on stderr if walk >0.5s and TTY-attached. Respect `NO_COLOR`. +- Auto-detect "this pattern looks pre-expanded by the shell" (positional has no `*`/`?` and multiple positionals) and emit a friendly warning. Keep the current epilog quoting note. +- Snapshot tests with `syrupy` covering `find`, `lcount`, `tsize`, and the new output formats. +- Shell completion: document `rglob --install-completion {bash,zsh,fish,powershell}`. +- Update `pyproject.toml` deps: add `typer`, `rich`. Update `docs/cli.md` to use **`mkdocs-typer2`** (`::: mkdocs-typer2` directive pointed at the Typer app; no Click bridge required). +- README: rewrite the CLI section with the new flags, output formats, and a section on shell completion. Keep the legacy CLI examples working — they still do. +- Extend `docs/architecture.md` with the CLI command hierarchy diagram (Mermaid `graph TD`). + +**Critical files**: `src/rglob/cli.py`, `tests/test_cli.py` (new, syrupy-based), `docs/cli.md`, `docs/architecture.md`, `pyproject.toml`, `README.md`. + +**Acceptance**: every existing CLI invocation in the README produces identical stdout (regression check); new flags work; `rglob find "*.py" --json | jq` round-trips; snapshot suite covers all subcommand × format combinations; shell completion installs and tab-completes flags; **aggregated coverage stays at 100%**. + +--- + +## Phase 5 — Fun features + +The labor-of-love payoff. Each subcommand is a self-contained mini-project. + +**Tasks** +- Tier 2 core filters wired into both `find()` and CLI: + - `min_size`/`max_size` accepting `int | str` ("1M", "10K"). Tiny size-string parser; reuse existing `kilobytes`/`megabytes`/`gigabytes`/`terabytes` for the math. + - `newer_than`/`older_than` accepting `datetime | timedelta | str` ("7d", "2024-01-01"). + - `kinds: set[Literal["f","d","l","x"]]` (file/dir/symlink/executable). +- `.gitignore` awareness: optional dep on `pathspec` (mature, small). `respect_gitignore: bool = False`. Walk `.gitignore` files top-down, merge into per-directory `PathSpec`, prune at `scandir` time. `pyproject.toml`: `[project.optional-dependencies] gitignore = ["pathspec"]`. CLI flag promoted from Phase 4 stub. +- New subcommands: + - **`stats`**: combined count + total size + line-count + extension breakdown in a `rich.table`. + - **`tree`**: ASCII/Unicode tree built on `rich.tree`, honours all filters. + - **`top`**: top-N largest files (default N=10), table output. + - **`dupes`**: duplicate-file detection. Strategy: bucket by size → bucket by `xxhash.xxh3_64` of first 4KB → full `xxh3_64` of survivors. Optional dep on `xxhash` (under the `[ext]` extra); fallback to `hashlib.file_digest(..., "blake2b")` when `xxhash` isn't installed (stdlib, no extra dep, fast enough to be a credible default in 2026). Output groups in a table. +- Benchmarks suite with `pytest-benchmark`: synthetic 10k-file tree, measure walk time, `--gitignore` overhead, `dupes` runtime. Upload as CI artifact (don't gate). +- Docs page per new subcommand with screenshots/asciinema; extend `docs/architecture.md` with the `dupes` pipeline (`flowchart LR`) and `.gitignore` pruning sequence diagrams in Mermaid. +- README: add a "Fun features" section showcasing `stats`/`tree`/`top`/`dupes` with example output snippets. + +**Critical files**: `src/rglob/rglob.py` (filters), `src/rglob/cli.py` (new subcommands), `src/rglob/_filters.py` (new size/time parsers), `src/rglob/_dupes.py` (new), `tests/test_dupes.py`, `tests/test_filters.py`, `bench/test_walk.py` (new), `docs/cli/*.md`, `docs/architecture.md`, `README.md`. + +**Acceptance**: each new subcommand has a docs page, snapshot tests, and a benchmark entry; `--gitignore` round-trips against a real `.gitignore` from this repo (it's a fun dogfood); **aggregated coverage stays at 100%** (every branch in `_dupes.py` and the new filters exercised). + +--- + +## Phase 6 — 2.0 cleanup & release + +The only release point. Everything from Phases 1–5 ships in one `2.0.0` PyPI publish. + +**Tasks** +- **Breaking**: `rglob()` and `rglob_()` now return `list[Path]` (was `list[str]`). One-line migration: `[str(p) for p in rglob(...)]`. Document loudly in CHANGELOG + a "Migrating to 2.0" page. +- Remove any deprecation shims accumulated in Phases 3–5. +- Optional Tier 3 stretch (only if still fun): `wcmatch`-style extended globs (`+(...)`, `@(...)`, `!(...)`) gated behind `extended: bool = False` and `pip install rglob[ext]`. +- Bump trove classifier to `Development Status :: 6 - Mature`. +- Flip `__version__` from `"2.0.0.dev0"` → `"2.0.0"`. +- Convert the `[Unreleased]` CHANGELOG section to `[2.0.0] – `; add the "Migrating to 2.0" docs page. +- Extend `docs/architecture.md` with the final public-API class diagram (Mermaid `classDiagram`) showing the 2.0 surface. +- README: final accuracy pass — Quick Start, API reference link, CLI reference link, "Migrating to 2.0" link, refreshed badges (e.g., bump PyPI badge to point at 2.0). +- Tag `v2.0.0` → `release.yml` workflow does the PyPI publish + GitHub Release. + +**Critical files**: `src/rglob/rglob.py`, `src/rglob/__init__.py`, `CHANGELOG.md`, `docs/migrating-to-2.0.md`, `docs/architecture.md`, `README.md`. + +**Acceptance**: clean `mypy --strict`; all tests pass; **aggregated coverage = 100%**; `pip install rglob==2.0.0` from PyPI works; `from rglob import rglob; rglob("/tmp", "*")` returns `list[Path]`; README banner + badges + content all reflect the shipped 2.0 surface. + +--- + +## Critical files (cross-phase quick reference) + +- `pyproject.toml` — focal point of Phases 1, 2, 4, 5 (PEP 621 metadata; hatch; ruff/mypy/pytest/coverage config; `[project.optional-dependencies]` for `dev`/`bdd`/`gitignore`/`ext`/`docs`; Typer + Rich + xxhash + pathspec deps) +- `src/rglob/rglob.py` — rewritten in Phase 3 onto `os.scandir`; gains `find`/`find_all`; new filters in Phases 3 & 5; legacy `rglob`/`rglob_`/`lcount`/`tsize` become wrappers +- `src/rglob/cli.py` — Phase 4 Typer migration; Phase 5 new subcommands (`stats`/`tree`/`top`/`dupes`) +- `src/rglob/__init__.py` — version single-source (Phase 1); export `find`/`find_all` (Phase 3); prune deprecated aliases (Phase 6) +- `.github/workflows/` — replace `pylint.yml` with `ci.yml` + `release.yml` + `docs.yml` (Phase 2); all gating +- `features/` — kept as parallel BDD suite; `features/steps/steps.py` refactored in Phase 2 to share helpers with `tests/` +- `tests/` — new dir in Phase 2; grows each phase; 100%-coverage discipline enforced locally via `--cov-fail-under=100` and on CI via the Codecov merged-report gate (per-job floor 95%) +- `docs/` — new in Phase 2; `index.md`, `api.md`, `cli.md`, `architecture.md` (Mermaid diagrams accreting per phase), `decisions/`, `contributing.md`, `plans/` (this file lives here) +- `README.md` — refreshed every phase; never goes stale +- `Makefile` — `help` (default), `build`, `lint`, `test`, `fmt`, `docs`, `dev-setup`, `clean` + +**Existing code worth reusing**: +- `kilobytes`/`megabytes`/`gigabytes`/`terabytes` in `src/rglob/rglob.py:11-32` — reuse directly for the Phase 5 size-string parser. +- The behave scenarios in `features/rglob.feature` — port their assertions into pytest in Phase 2; the existing fixture pattern (`create_root_dir`, `create_subdirectories`, `create_files`) translates cleanly to `tmp_path` fixtures. + +--- + +## Verification + +End-to-end checks per phase. No PyPI publishes happen until Phase 6; everything before that is verified locally and in CI only. + +**Phase 1**: `pip install -e . && python -c "import rglob; print(rglob.__version__)"` → `2.0.0.dev0`. `git status` clean of `egg-info`. `CODE_OF_CONDUCT.md` exists (required by `pyproject.toml` sdist `include`). `README.md` opens with an ASCII banner that renders cleanly in a GitHub-flavoured markdown preview. + +**Phase 2**: `make help` prints every target with descriptions; `make lint test build docs` green locally. PR shows CI matrix green across `3.11–3.14` × `ubuntu/macos/windows`. Codecov merged report shows **100%** coverage; the local `--cov-fail-under=100` gate fails a deliberately-broken test that drops a line uncovered. Branch protection blocks a PR with a failing test. Docs site at `https://chris-piekarski.github.io/python-rglob/` renders Mermaid diagrams on `architecture.md`. Dry-run `release.yml` against TestPyPI. + +**Phase 3**: Every existing behave scenario passes unchanged (`behave`). New pytest+hypothesis suite passes (`pytest`). Coverage still 100%. Benchmark: `pytest bench/ --benchmark-only` shows `find()` ≥2× faster than legacy `rglob()` on a 10k-file tree. `mypy --strict src/rglob` clean. README Quick Start now demonstrates `find()` and the `**` recursive form actually works. + +**Phase 4**: Every README CLI example produces identical stdout (regression check). `rglob find "*.py" --json | jq '.[] | .path'` round-trips. `rglob --install-completion bash && exec bash && rglob f` completes to `find`. Snapshot suite (`pytest tests/test_cli.py`) green. Coverage still 100%. Architecture diagram for CLI hierarchy renders on the docs site. + +**Phase 5**: `rglob stats "*.py"` shows a populated `rich.table`. `rglob tree --max-depth 3` renders a tree. `rglob dupes ~/Downloads` correctly groups two intentionally-duplicated test files. `rglob find "*" --gitignore` against this repo excludes `dist/`, `.venv/`, etc. Aggregated coverage still 100% — every branch in `_dupes.py` and the new filters is exercised. README "Fun features" section showcases each new subcommand with copy-paste-able examples. + +**Phase 6**: `pip install rglob==2.0.0 && python -c "from rglob import rglob; from pathlib import Path; assert all(isinstance(p, Path) for p in rglob('.', '*'))"` passes. Migration docs page exists at `https://chris-piekarski.github.io/python-rglob/migrating-to-2.0/`. Final aggregated coverage report is 100%. README banner + badges + content reflect the shipped 2.0 surface. PyPI listing shows v2.0.0. diff --git a/docs/plans/rglob-2.0-pr-body.md b/docs/plans/rglob-2.0-pr-body.md new file mode 100644 index 0000000..5e4e365 --- /dev/null +++ b/docs/plans/rglob-2.0-pr-body.md @@ -0,0 +1,90 @@ +# rglob 2.0 — PR Summary + +This page mirrors the body of [PR #6](https://github.com/chris-piekarski/python-rglob/pull/6), +preserved in-repo for historical reference. The PR description on GitHub +uses repo-root paths; this page uses doc-tree-relative paths so it renders +correctly under MkDocs. + +## Summary + +Top-to-bottom modernization shipping **rglob 2.0**, delivered across six +phases per [`modernization-roadmap.md`](modernization-roadmap.md): + +1. **Packaging hygiene** — PEP 621 + hatchling, single-sourced `__version__`, + `src/` layout, full doc set (CHANGELOG / CONTRIBUTING / SECURITY / + CODE_OF_CONDUCT / AGENTS), ASCII-banner README with badges. +2. **Tooling overhaul** — ruff (lint + format) replaces pylint, mypy + `--strict` with `py.typed`, pytest as primary suite (behave kept as + parallel BDD job per [ADR-0007](../decisions/0007-behave-parallel.md)), + aggregated 100% coverage via Codecov + ([ADR-0006](../decisions/0006-aggregated-coverage.md)), + three GitHub workflows (`ci.yml` 3.11–3.14 × ubuntu/macos/windows, + `release.yml` OIDC trusted publishing, `docs.yml` gh-pages), MkDocs + Material site with Mermaid diagrams, Makefile, pre-commit hooks, + renovate, codecov. +3. **Core API on `os.scandir`** — new `find()` / `find_all()` with filters + for `exclude` / `max_depth` / `hidden` / `follow_symlinks` / + `case_sensitive` / `sort` / `on_error`, `**` recursive globs, + symlink-loop detection via realpath memo + ([ADR-0008](../decisions/0008-security-model.md)), + deterministic sort-by-default, OS-aware case sensitivity, hypothesis + property tests. +4. **Typer + Rich CLI** — byte-compatible `find` / `lcount` / `tsize`, + plus new filter flags (`-E`, `-d`, `-H`, `-L`, case toggle), output + formats (`--json`, `--jsonl`, `-0`, `--format`), shell pre-expansion + detection, `--install-completion`, syrupy snapshot tests. +5. **Fun features** — new `stats` / `tree` / `top` / `dupes` subcommands; + duplicate detection (size → 4-KiB hash → full hash) using + `xxhash.xxh3_64` with BLAKE2b fallback; `.gitignore` awareness via + pathspec; `--type` / `--min-size` / `--max-size` / `--newer-than` / + `--older-than` filters. +6. **2.0 cleanup** — `__version__ = "2.0.0"`, `Development Status :: 6 - Mature`, + migration guide at [`migrating-to-2.0.md`](../migrating-to-2.0.md), + final public-API class diagram. + +## Breaking change + +- **`rglob()` and `rglob_()` now return `list[Path]`** (was `list[str]` in + 1.x). One-line migration: + + ```python + paths = [str(p) for p in rglob(base, pattern)] + ``` + + See [migrating-to-2.0](../migrating-to-2.0.md) and + [ADR-0003](../decisions/0003-path-return.md) for full context. + +## Decisions captured as ADRs + +- [0001 — build backend (hatchling)](../decisions/0001-build-backend.md) +- [0002 — Python 3.11 floor](../decisions/0002-python-floor.md) +- [0003 — `list[Path]` return at 2.0](../decisions/0003-path-return.md) +- [0004 — Typer + Rich for the CLI](../decisions/0004-typer-rich.md) +- [0005 — single 2.0.0 release strategy](../decisions/0005-single-release.md) +- [0006 — aggregated 100% coverage](../decisions/0006-aggregated-coverage.md) +- [0007 — behave kept as parallel BDD](../decisions/0007-behave-parallel.md) +- [0008 — security model](../decisions/0008-security-model.md) + +## Test plan + +- [x] `make lint` — ruff (lint + format) + mypy `--strict` clean +- [x] `make test` — pytest 119/119 passed @ **100%** local coverage, behave + 7/7 scenarios +- [x] `make build` — `hatch build` produces `rglob-2.0.0.tar.gz` and + `rglob-2.0.0-py3-none-any.whl` +- [x] `make docs-build` — `mkdocs build --strict` clean +- [x] End-to-end CLI smoke (`rglob find --json`, `rglob stats`, + `rglob dupes` with planted duplicates) all return expected output +- [ ] CI matrix (3.11–3.14 × ubuntu/macos/windows) — to confirm on the PR +- [ ] Codecov merged report hits 100% — to confirm on the PR +- [ ] Docs site renders on gh-pages — to confirm post-merge +- [ ] Dry-run `release.yml` against TestPyPI — to confirm separately before + cutting `v2.0.0` + +## Notes + +- No tag pushed; the actual `v2.0.0` release is a follow-up (per the + single-release strategy in + [ADR-0005](../decisions/0005-single-release.md)). +- Branch protection on `master` should be enabled in repo settings before + merge (CONTRIBUTING.md documents the required checks). diff --git a/features/agent_cli.feature b/features/agent_cli.feature new file mode 100644 index 0000000..aee7e3f --- /dev/null +++ b/features/agent_cli.feature @@ -0,0 +1,45 @@ +Feature: New agent-friendly CLI commands + As a developer or coding agent + I want to use the new rglob CLI commands (grep, count, describe, schema, capabilities) + So that I can perform structured searches and introspect the tool reliably + + Background: + Given I create a root directory + And I create 3 subdirectories in each directory + And I create 2 .py files in each directory + And I create 1 .txt file in each directory + + Scenario: Grep for content and receive structured JSON + When I run "rglob grep . --json" + Then the command succeeds + And the JSON output should contain a "results" list + And the JSON output should contain "truncated" and "errors" fields + + Scenario: Count lines with filters using the new count command + When I run "rglob count '*.py' --no-empty --no-comments --json" + Then the command succeeds + And the JSON output should contain "files", "lines", and "bytes" + + Scenario: Describe a subcommand + When I run "rglob describe find" + Then the command succeeds + And the output should be valid JSON + And the JSON should contain the key "arguments" + + Scenario: Retrieve JSON Schema for a subcommand + When I run "rglob schema grep" + Then the command succeeds + And the output should be valid JSON Schema Draft 2020-12 + And the JSON should contain an "$id" field + + Scenario: Capabilities reports installed features and versions + When I run "rglob capabilities --json" + Then the command succeeds + And the JSON should contain "agent_api_version" + And the JSON should contain "extras" + And the JSON should contain "predicates" + + Scenario: Grep respects --limit and reports truncation + When I run "rglob grep . --limit 1 --json" + Then the command succeeds + And the JSON output should indicate that results were truncated diff --git a/features/environment.py b/features/environment.py index c40ad81..ecf6eaa 100644 --- a/features/environment.py +++ b/features/environment.py @@ -1,4 +1,5 @@ """Behave environment hooks for test state setup/teardown.""" + from __future__ import annotations diff --git a/features/steps/steps.py b/features/steps/steps.py index 825740d..b1d2250 100644 --- a/features/steps/steps.py +++ b/features/steps/steps.py @@ -3,11 +3,17 @@ Pylint notes: - Behave's decorators confuse type inference; suppress not-callable for them. """ + # pylint: disable=missing-function-docstring,not-callable from __future__ import annotations +import json import os +import shlex +import subprocess +import sys import tempfile + from behave import given, then, when import rglob @@ -30,20 +36,15 @@ def create_subdirectories(context, num_of_sub_dirs: int) -> None: if not hasattr(context, "dirs") or not context.dirs: context.dirs = [] for _ in range(num_of_sub_dirs): - subdirs.append( - tempfile.mkdtemp(prefix="subdir_", suffix="_rglob", dir=context.root) - ) + subdirs.append(tempfile.mkdtemp(prefix="subdir_", suffix="_rglob", dir=context.root)) else: for root_dir in context.dirs: for _ in range(num_of_sub_dirs): - subdirs.append( - tempfile.mkdtemp( - prefix="subdir_", suffix="_rglob", dir=root_dir - ) - ) + subdirs.append(tempfile.mkdtemp(prefix="subdir_", suffix="_rglob", dir=root_dir)) context.dirs.extend(subdirs) +@given("I create {num_of_files:d} {file_type} file in each directory") @given("I create {num_of_files:d} {file_type} files in each directory") def create_files(context, num_of_files: int, file_type: str) -> None: if not hasattr(context, "known_sizes"): @@ -70,25 +71,21 @@ def sum_filesize(context, file_type: str) -> None: @then("I can find the same size for {file_type}") def find_total_size(context, file_type: str) -> None: found_total_size = rglob.tsize(context.root, f"*{file_type}", rglob.kilobytes) - assert ( - found_total_size == context.known_size - ), f"Known size {context.known_size} doesn't match found size {found_total_size}" + assert found_total_size == context.known_size, ( + f"Known size {context.known_size} doesn't match found size {found_total_size}" + ) @then("I find {expected_num_of_dirs:d} total directories") def find_directories(context, expected_num_of_dirs: int) -> None: matches = rglob.rglob(context.root, "*_rglob") - assert ( - len(matches) == expected_num_of_dirs - ), f"Found {len(matches)} directories" + assert len(matches) == expected_num_of_dirs, f"Found {len(matches)} directories" @then("I find {expected_num_of_files:d} total {file_type} files") def find_files(context, expected_num_of_files: int, file_type: str) -> None: matches = rglob.rglob(context.root, f"*{file_type}") - assert ( - len(matches) == expected_num_of_files - ), f"Found {len(matches)} files" + assert len(matches) == expected_num_of_files, f"Found {len(matches)} files" @then("I delete all") @@ -103,32 +100,144 @@ def delete_all_directories(context) -> None: context.dirs = [] context.known_sizes = {} + @given("I add {num_lines:d} lines to each {file_type} file") def add_lines_to_files(context, num_lines: int, file_type: str) -> None: all_files = rglob.rglob(context.root, f"*{file_type}") for f in all_files: with open(f, "w", encoding="utf-8") as file: for i in range(num_lines): - file.write(f"line {i+1}\n") + file.write(f"line {i + 1}\n") + @when("I count the lines in all {file_type} files") def count_lines_in_files(context, file_type: str) -> None: context.line_count = rglob.lcount(context.root, f"*{file_type}") + @then("I should find {expected_line_count:d} lines") def check_line_count(context, expected_line_count: int) -> None: assert context.line_count == expected_line_count, ( f"Expected {expected_line_count} lines, but found {context.line_count}" ) + @when("I change the current working directory to the root directory") def change_cwd_to_root(context) -> None: os.chdir(context.root) + @when("I use rglob_ to find all {file_type} files") def use_rglob_(context, file_type: str) -> None: context.found_files = rglob.rglob_(f"*{file_type}") + +# ============================================================ +# New steps for agent platform CLI commands (grep, count, describe, schema, capabilities) +# ============================================================ + + +@when('I run "{command}"') +def run_rglob_command(context, command: str): + """Run a rglob CLI command via python -m for reliability in test environments.""" + args = shlex.split(command) + if args and args[0] == "rglob": + args = args[1:] + result = subprocess.run( + [sys.executable, "-m", "rglob.cli", *args], + cwd=context.root, + capture_output=True, + text=True, + timeout=30, + ) + context.last_result = result + context.last_stdout = result.stdout + context.last_stderr = result.stderr + + +@then("the command succeeds") +def command_succeeds(context): + assert context.last_result.returncode == 0, ( + f"Command failed (exit code {context.last_result.returncode})\n" + f"stderr:\n{context.last_stderr}\n" + f"stdout:\n{context.last_stdout}" + ) + + +@then("the output should be valid JSON") +def output_is_valid_json(context): + try: + json.loads(context.last_stdout) + except json.JSONDecodeError as exc: + raise AssertionError( + f"Output was not valid JSON: {exc}\nOutput was:\n{context.last_stdout}" + ) from exc + + +def _last_json(context): + """Return the most recent CLI stdout as JSON.""" + return json.loads(context.last_stdout) + + +def _contains_key(value, key: str) -> bool: + """Return true if a JSON-compatible value contains a key anywhere.""" + if isinstance(value, dict): + return key in value or any(_contains_key(item, key) for item in value.values()) + if isinstance(value, list): + return any(_contains_key(item, key) for item in value) + return False + + +@then('the JSON output should contain a "{key}" list') +def json_contains_list(context, key: str): + data = _last_json(context) + assert key in data and isinstance(data[key], list), ( + f"Expected '{key}' to be a list in JSON output" + ) + + +@then('the JSON output should contain "{field}" and "errors" fields') +def json_contains_truncation_and_errors(context, field: str): + data = _last_json(context) + assert field in data, f"Missing field: {field}" + assert "errors" in data, "Missing 'errors' field" + + +@then('the JSON output should contain "{field1}", "{field2}", and "{field3}"') +def json_contains_three_fields(context, field1: str, field2: str, field3: str): + data = _last_json(context) + for field in (field1, field2, field3): + assert field in data, f"Missing field: {field}" + + +@then('the JSON should contain the key "{key}"') +@then('the JSON should contain "{key}"') +def json_contains_key(context, key: str): + data = _last_json(context) + assert key in data, f"Expected key '{key}' not found in JSON output" + + +@then("the output should be valid JSON Schema Draft 2020-12") +def output_is_valid_json_schema(context): + data = _last_json(context) + if "$schema" in data: + assert data["$schema"] == "https://json-schema.org/draft/2020-12/schema" + return + for schema in data.values(): + assert schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" + + +@then('the JSON should contain an "{key}" field') +def json_contains_key_anywhere(context, key: str): + assert _contains_key(_last_json(context), key), f"Expected key '{key}' not found" + + +@then("the JSON output should indicate that results were truncated") +def json_indicates_truncation(context): + data = _last_json(context) + assert data.get("truncated") is True, "Expected 'truncated' to be True in JSON output" + + @then("I should find {expected_num_files:d} files") def check_found_files(context, expected_num_files: int) -> None: assert len(context.found_files) == expected_num_files, ( diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..5fcd261 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,104 @@ +site_name: rglob +site_description: Lightweight recursive glob helpers for Python. +site_url: https://chris-piekarski.github.io/python-rglob/ +repo_url: https://github.com/chris-piekarski/python-rglob +repo_name: chris-piekarski/python-rglob +edit_uri: edit/master/docs/ + +theme: + name: material + features: + - navigation.tabs + - navigation.sections + - navigation.expand + - navigation.indexes + - navigation.top + - content.code.copy + - content.action.edit + - content.tabs.link + - toc.follow + palette: + - media: "(prefers-color-scheme: light)" + scheme: default + primary: indigo + accent: indigo + toggle: + icon: material/weather-sunny + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: indigo + accent: indigo + toggle: + icon: material/weather-night + name: Switch to light mode + +nav: + - Home: index.md + - API: api.md + - CLI: cli.md + - Agents: + - agents/index.md + - "Python API": agents/python-api.md + - "MCP setup": agents/mcp-setup.md + - "CLI recipes": agents/cli-recipes.md + - Stability: agents/stability.md + - Safety: agents/safety.md + - Architecture: architecture.md + - Decisions: + - decisions/index.md + - "Build backend: hatchling": decisions/0001-build-backend.md + - "Python 3.11 floor": decisions/0002-python-floor.md + - "rglob returns list[Path] at 2.0": decisions/0003-path-return.md + - "Typer + Rich for CLI": decisions/0004-typer-rich.md + - "Single release strategy": decisions/0005-single-release.md + - "Aggregated 100% coverage": decisions/0006-aggregated-coverage.md + - "Behave kept as parallel BDD suite": decisions/0007-behave-parallel.md + - "Security model": decisions/0008-security-model.md + - "Agent API contract": decisions/0009-agent-api-contract.md + - "Agent safety model": decisions/0010-agent-safety-model.md + - Contributing: contributing.md + - Examples: + - "Agent recipes": examples/agent-recipes.md + - Plans: + - "Modernization roadmap": plans/modernization-roadmap.md + - "2.0 PR summary": plans/rglob-2.0-pr-body.md + - "Agent platform additions": plans/agent-platform.md + - "Migrating to 2.0": migrating-to-2.0.md + - Changelog: changelog.md + +markdown_extensions: + - admonition + - attr_list + - md_in_html + - toc: + permalink: true + - pymdownx.details + - pymdownx.tabbed: + alternate_style: true + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.snippets: + check_paths: true + base_path: ["."] + +plugins: + - search + - mkdocstrings: + handlers: + python: + paths: [src] + options: + show_source: true + docstring_style: google + heading_level: 2 + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/chris-piekarski/python-rglob + - icon: fontawesome/brands/python + link: https://pypi.org/project/rglob/ diff --git a/pyproject.toml b/pyproject.toml index 272ae4e..1dd7c2b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,196 @@ -[tool.pylint.main] -recursive = true +[build-system] +requires = ["hatchling>=1.21"] +build-backend = "hatchling.build" -[tool.pylint.messages_control] -disable = [] +[project] +name = "rglob" +description = "Lightweight recursive glob helpers for Python — find files, count lines, sum sizes." +readme = "README.md" +requires-python = ">=3.11" +license = "Apache-2.0" +license-files = ["LICENSE"] +keywords = ["glob", "rglob", "recursive", "filesystem", "find", "line-counter"] +authors = [ + { name = "Christopher Piekarski", email = "chris@cpiekarski.com" }, +] +maintainers = [ + { name = "Christopher Piekarski", email = "chris@cpiekarski.com" }, +] +classifiers = [ + "Development Status :: 6 - Mature", + "Environment :: Console", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: System :: Filesystems", + "Topic :: Utilities", + "Typing :: Typed", +] +dynamic = ["version"] +dependencies = [ + "typer>=0.12", + "rich>=13", +] -[tool.pylint.format] -max-line-length = 100 +[project.urls] +Homepage = "https://github.com/chris-piekarski/python-rglob" +Repository = "https://github.com/chris-piekarski/python-rglob" +Issues = "https://github.com/chris-piekarski/python-rglob/issues" +Changelog = "https://github.com/chris-piekarski/python-rglob/blob/master/CHANGELOG.md" +Documentation = "https://chris-piekarski.github.io/python-rglob/" +[project.scripts] +rglob = "rglob.cli:main" + +[project.optional-dependencies] +dev = [ + "ruff>=0.6", + "mypy>=1.10", + "pytest>=8", + "pytest-cov>=5", + "pytest-asyncio>=0.23", + "hypothesis>=6.100", + "syrupy>=4", + "pre-commit>=3", + "twine>=5", +] +bdd = [ + "behave>=1.2.6", +] +docs = [ + "mkdocs>=1.6", + "mkdocs-material>=9.5", + "mkdocstrings[python]>=0.25", + "mkdocs-typer2>=0.3", + "pymdown-extensions>=10", +] +gitignore = [ + "pathspec>=0.12", +] +ext = [ + "xxhash>=3.4", + "wcmatch>=8.5", +] +mcp = [ + "mcp>=1.0", + "pathspec>=0.12", + "xxhash>=3.4", +] +bench = [ + "pytest-benchmark>=4", +] + +[tool.hatch.version] +path = "src/rglob/__init__.py" + +[tool.hatch.build.targets.wheel] +packages = ["src/rglob"] + +[tool.hatch.build.targets.sdist] +include = [ + "src/", + "bench/", + "scripts/", + "tests/", + "features/", + "docs/", + "README.md", + "CHANGELOG.md", + "CONTRIBUTING.md", + "SECURITY.md", + "CODE_OF_CONDUCT.md", + "LICENSE", + "Makefile", +] + +# ─── Ruff ────────────────────────────────────────────────────────────────────── +[tool.ruff] +line-length = 100 +target-version = "py311" +src = ["src", "tests"] +extend-exclude = [".venv", "site", "dist", "build"] + +[tool.ruff.lint] +select = [ + "E", "F", "W", # pyflakes / pycodestyle + "I", # isort + "B", # bugbear + "UP", # pyupgrade + "SIM", # simplify + "RUF", # ruff-native + "PTH", # pathlib instead of os.path + "PERF", # perflint + "D", # pydocstyle +] +ignore = [ + "D100", # missing docstring in public module — we have selective coverage + "D104", # missing docstring in public package + "D203", # one-blank-line-before-class conflicts with D211 + "D213", # multi-line-summary-second-line conflicts with D212 +] + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.ruff.lint.per-file-ignores] +"tests/**" = ["D", "B011"] +# Typer's idiomatic defaults use `Path.cwd()` and `[]` literals; these are +# *not* mutable-default footguns (Click rebinds them per call), so B008/B006 +# are suppressed for the CLI module. +"src/rglob/cli.py" = ["B006", "B008"] +"features/**" = ["D", "PTH", "PERF"] +"bench/**" = ["D"] + +# ─── Mypy ────────────────────────────────────────────────────────────────────── +[tool.mypy] +python_version = "3.11" +strict = true +files = ["src/rglob"] +show_error_codes = true +warn_unused_ignores = true +enable_error_code = ["redundant-self", "redundant-expr", "truthy-bool"] + +[[tool.mypy.overrides]] +module = ["behave.*", "pathspec.*", "xxhash.*"] +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = ["mcp.*"] +ignore_missing_imports = true + +# ─── Pytest ──────────────────────────────────────────────────────────────────── +[tool.pytest.ini_options] +minversion = "8.0" +testpaths = ["tests"] +addopts = [ + "-ra", + "--strict-markers", + "--strict-config", +] +xfail_strict = true +filterwarnings = ["error"] + +# ─── Coverage ────────────────────────────────────────────────────────────────── +[tool.coverage.run] +source = ["rglob"] +branch = true +parallel = true + +[tool.coverage.paths] +source = ["src/rglob", "*/site-packages/rglob"] + +[tool.coverage.report] +fail_under = 100 +show_missing = true +skip_covered = false +exclude_also = [ + "if __name__ == .__main__.:", + "raise NotImplementedError", + "if TYPE_CHECKING:", + "@overload", +] diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..d4e0ab9 --- /dev/null +++ b/renovate.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:recommended", + ":semanticCommits", + ":dependencyDashboard", + "schedule:weekly" + ], + "labels": ["dependencies"], + "pre-commit": { + "enabled": true + }, + "packageRules": [ + { + "matchUpdateTypes": ["minor", "patch"], + "matchCurrentVersion": "!/^0/", + "automerge": false + } + ] +} diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 78df6b8..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,3 +0,0 @@ -pylint -behave - diff --git a/rglob/__init__.py b/rglob/__init__.py deleted file mode 100644 index be89892..0000000 --- a/rglob/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Public API for rglob package.""" - -from rglob.rglob import * # noqa: F401,F403 re-export convenience - -__all__ = [ - # re-exported from rglob.rglob - "rglob", - "rglob_", - "lcount", - "tsize", - "kilobytes", - "megabytes", - "gigabytes", - "terabytes", -] diff --git a/rglob/cli.py b/rglob/cli.py deleted file mode 100644 index 20583bc..0000000 --- a/rglob/cli.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Command-line interface for rglob helpers.""" - -import argparse -import os -from rglob import ( - rglob, - lcount, - tsize, - kilobytes, - megabytes, - gigabytes, - terabytes, -) - -def main() -> None: - """Parse arguments and execute rglob CLI commands.""" - - parser = argparse.ArgumentParser( - description="Recursive file operations.", - epilog=( - "Note: Quote or escape glob patterns so your shell doesn't pre-expand " - "them. For example, use \"*.py\" (with quotes), not *.py." - ), - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - subparsers = parser.add_subparsers(dest="command", required=True) - - # Find command - find_parser = subparsers.add_parser("find", help="Find files recursively.") - find_parser.add_argument("pattern", help="The glob pattern to match.") - find_parser.add_argument( - "--base", default=os.getcwd(), help="The base directory to search in." - ) - - # Line count command - lcount_parser = subparsers.add_parser( - "lcount", help="Count lines in files recursively." - ) - lcount_parser.add_argument("pattern", help="The glob pattern to match.") - lcount_parser.add_argument( - "--base", default=os.getcwd(), help="The base directory to search in." - ) - lcount_parser.add_argument( - "--no-empty", action="store_true", help="Exclude empty lines." - ) - lcount_parser.add_argument( - "--no-comments", - action="store_true", - help="Exclude comment lines (starting with #).", - ) - - # Total size command - tsize_parser = subparsers.add_parser( - "tsize", help="Calculate the total size of files recursively." - ) - tsize_parser.add_argument("pattern", help="The glob pattern to match.") - tsize_parser.add_argument( - "--base", default=os.getcwd(), help="The base directory to search in." - ) - tsize_parser.add_argument( - "--unit", - default="mb", - choices=["kb", "mb", "gb", "tb"], - help="The unit to display the size in.", - ) - - args = parser.parse_args() - - if args.command == "find": - files = rglob(args.base, args.pattern) - for f in files: - print(f) - elif args.command == "lcount": - filter_funcs = [] - if args.no_empty: - filter_funcs.append(lambda line: line.strip()) - if args.no_comments: - filter_funcs.append(lambda line: not line.strip().startswith("#")) - - def combined_filter(line): - return all(f(line) for f in filter_funcs) - - count = lcount( - args.base, - args.pattern, - func=combined_filter if filter_funcs else (lambda _line: True), - ) - print(f"Total lines: {count}") - elif args.command == "tsize": - unit_funcs = { - "kb": kilobytes, - "mb": megabytes, - "gb": gigabytes, - "tb": terabytes, - } - total_size = tsize(args.base, args.pattern, func=unit_funcs[args.unit]) - print(f"Total size: {total_size:.2f} {args.unit.upper()}") - -if __name__ == "__main__": - main() diff --git a/rglob/rglob.py b/rglob/rglob.py deleted file mode 100644 index e2e4415..0000000 --- a/rglob/rglob.py +++ /dev/null @@ -1,127 +0,0 @@ -"""Lightweight recursive glob helpers for files, line counts, and sizes.""" - -from __future__ import annotations - -import glob -import math -import os -from typing import Callable, Iterable, List - - -def kilobytes(value: float) -> float: - """Convert bytes to kilobytes (KiB).""" - - return value / math.pow(2.0, 10.0) - - -def megabytes(value: float) -> float: - """Convert bytes to megabytes (MiB).""" - - return value / math.pow(2.0, 20.0) - - -def gigabytes(value: float) -> float: - """Convert bytes to gigabytes (GiB).""" - - return value / math.pow(2.0, 30.0) - - -def terabytes(value: float) -> float: - """Convert bytes to terabytes (TiB).""" - - return value / math.pow(2.0, 40.0) - - -def _get_dirs(base: str) -> List[str]: - """Return immediate sub-paths under base that are directories.""" - - return [ - path for path in glob.iglob(os.path.join(base, "*")) if os.path.isdir(path) - ] - - -def _count(files: Iterable[str], func: Callable[[str], bool]) -> int: - """Count lines across files that satisfy predicate func.""" - - total = 0 - for path in files: - with open(path, "r", encoding="utf-8", errors="ignore") as handle: - total += sum(1 for line in handle if func(line)) - return total - - -def _sum(files: Iterable[str]) -> int: - """Sum sizes of file paths (ignores directories).""" - - total_size = 0 - for path in files: - if not os.path.isdir(path): - total_size += os.path.getsize(path) - return total_size - - -def rglob(base: str, pattern: str) -> list[str]: - """Recursive glob starting in specified directory.""" - - matches: list[str] = [] - matches.extend(glob.glob(os.path.join(base, pattern))) - dirs = _get_dirs(base) - if dirs: - for dpath in dirs: - matches.extend(rglob(dpath, pattern)) - return matches - - -def rglob_(pattern: str) -> list[str]: - """Recursive glob using the current working directory as base.""" - - return rglob(os.getcwd(), pattern) - - -def lcount( - base: str, pattern: str, func: Callable[[str], bool] = lambda _line: True -) -> int: - """Count number of lines across files found matching pattern. - - Params: - base: root directory to start the search - pattern: pattern for glob to match (e.g. "*.py") - func: boolean filter function applied per line - example: lambda line: bool(line.strip()) # don't count empty lines - default: lambda line: True - """ - - all_files = rglob(base, pattern) - return _count(all_files, func) - - -def tsize(base: str, pattern: str, func: Callable[[float], float] = megabytes) -> float: - """Sum and return the total size of every file found from glob(base, pattern). - - Params: - base: root directory to start the search - pattern: pattern for glob to match (e.g. "*.py") - func: unit prefix conversion function - example: lambda x: x / math.pow(2.0, 20.0) # megabytes - default: megabytes - """ - - all_files = rglob(base, pattern) - total_size = _sum(all_files) - return func(total_size) - - -if __name__ == "__main__": - # Filter out empty lines and comments. - def filter_func(line: str) -> bool: - """Return True for non-empty, non-comment lines.""" - return bool(line.strip()) and not line.strip().startswith("#") - - py_files = rglob(os.path.dirname(__file__), "*.py") - print(f"{_count(py_files, filter_func)} total lines") - - py_files_cwd = rglob_("*.py") - print(f"{_count(py_files_cwd, filter_func)} total lines") - print( - f"{lcount(os.path.dirname(__file__), '*.py', filter_func)} total lines" - ) diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e3d9df4 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""Repository maintenance scripts.""" diff --git a/scripts/generate_repo_stats.py b/scripts/generate_repo_stats.py new file mode 100644 index 0000000..2a9145c --- /dev/null +++ b/scripts/generate_repo_stats.py @@ -0,0 +1,248 @@ +"""Generate basic repository stats and update OVERVIEW.md. + +Outputs a Markdown section with a Mermaid pie chart and summary tables +between markers in OVERVIEW.md: + + + ... generated ... + + +Run via the Makefile: + + make repo-stats + +Or directly: + + python -m scripts.generate_repo_stats +""" + +from collections.abc import Iterable +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +OVERVIEW_PATH = REPO_ROOT / "OVERVIEW.md" +PACKAGE_DIR = REPO_ROOT / "src" / "rglob" + +# Top-level directories that count as their own "area". Anything else falls +# into "other". +KNOWN_AREAS = {"src", "tests", "features", "docs", "scripts", "bench"} + +# Directories pruned from the scan entirely. +IGNORE_DIRS = { + ".git", + ".venv", + "venv", + "env", + "build", + "dist", + "site", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + ".idea", + ".vscode", + "__pycache__", + "htmlcov", + "node_modules", +} + +# File extensions counted, with their comment prefix (used so "code lines" +# excludes blank lines and comment-only lines). +COMMENT_PREFIX: dict[str, str] = { + ".py": "#", + ".sh": "#", + ".bash": "#", + ".yml": "#", + ".yaml": "#", + ".toml": "#", + ".ini": ";", + ".cfg": "#", + ".feature": "#", +} + +# Extensions where "non-blank line" is the closest analogue to "code". +CONSIDER_EXTS: set[str] = set(COMMENT_PREFIX) | {".md", ".txt", ".json"} + + +def is_ignored(path: Path) -> bool: + """Return True when path lies under an ignored directory.""" + return any(part in IGNORE_DIRS for part in path.parts) + + +def count_file(path: Path) -> tuple[int, int]: + """Return ``(total_lines, code_lines)`` for ``path``. + + ``code_lines`` excludes blank lines and lines whose first non-whitespace + character matches the language's comment prefix. For markup/data + formats (md/json/txt), every non-blank line counts as code. + """ + try: + text = path.read_text(encoding="utf-8", errors="ignore") + except OSError: + return 0, 0 + + total = 0 + code = 0 + prefix = COMMENT_PREFIX.get(path.suffix.lower()) + + for raw in text.splitlines(): + total += 1 + line = raw.strip() + if not line: + continue + if prefix and line.startswith(prefix): + continue + code += 1 + return total, code + + +def iter_repo_files(root: Path) -> Iterable[Path]: + """Yield considered files under ``root`` excluding ignored paths.""" + for p in root.rglob("*"): + if p.is_dir() or is_ignored(p): + continue + if p.suffix.lower() in CONSIDER_EXTS: + yield p + + +def aggregate_stats() -> dict[str, dict[str, dict[str, int]] | dict[str, int]]: + """Aggregate stats by top-level area and by `src/rglob` submodule.""" + by_area: dict[str, dict[str, int]] = {} + by_module_py: dict[str, dict[str, int]] = {} + + def bump(bucket: dict[str, dict[str, int]], key: str, total: int, code: int) -> None: + d = bucket.setdefault(key, {"total": 0, "code": 0}) + d["total"] += total + d["code"] += code + + for f in iter_repo_files(REPO_ROOT): + total, code = count_file(f) + parts = f.relative_to(REPO_ROOT).parts + area = parts[0] if parts else "other" + if area not in KNOWN_AREAS: + area = "other" + bump(by_area, area, total, code) + + # Module breakdown: Python files under `src/rglob/`. Files directly + # under the package root land in "root"; everything else groups by + # immediate child (file or subdir). + try: + rel = f.relative_to(PACKAGE_DIR) + except ValueError: + continue + if f.suffix == ".py": + mod = rel.parts[0] if len(rel.parts) > 1 else "root" + bump(by_module_py, mod, total, code) + + return { + "by_area": by_area, + "by_module_py": by_module_py, + "overall": { + "total": sum(v["total"] for v in by_area.values()), + "code": sum(v["code"] for v in by_area.values()), + }, + } + + +def format_number(n: int) -> str: + """Format ``n`` with thousands separators for Markdown output.""" + return f"{n:,}" + + +def generate_markdown(stats: dict[str, dict[str, dict[str, int]] | dict[str, int]]) -> str: + """Render repository statistics as a Markdown section.""" + by_area: dict[str, dict[str, int]] = stats["by_area"] # type: ignore[assignment] + by_module_py: dict[str, dict[str, int]] = stats["by_module_py"] # type: ignore[assignment] + overall: dict[str, int] = stats["overall"] # type: ignore[assignment] + + pie_lines = ["```mermaid", "pie title Code LOC by Area"] + for name, data in sorted(by_area.items(), key=lambda kv: kv[1]["code"], reverse=True): + if data["code"] == 0: + continue + pie_lines.append(f' "{name}" : {data["code"]}') + pie_lines.append("```") + + top_n = 8 + sorted_mods = sorted(by_module_py.items(), key=lambda kv: kv[1]["code"], reverse=True) + module_pie_lines = ["```mermaid", "pie title Code LOC by Module (src/rglob, Python only)"] + other_sum = sum(v["code"] for _, v in sorted_mods[top_n:]) + for name, data in sorted_mods[:top_n]: + if data["code"] == 0: + continue + module_pie_lines.append(f' "{name}" : {data["code"]}') + if other_sum > 0: + module_pie_lines.append(f' "other" : {other_sum}') + module_pie_lines.append("```") + + area_rows = ["| Area | Code LOC | Total Lines |", "|------|----------|-------------|"] + for name, data in sorted(by_area.items(), key=lambda kv: kv[1]["code"], reverse=True): + area_rows.append( + f"| {name} | {format_number(data['code'])} | {format_number(data['total'])} |" + ) + + module_rows = [ + "| Module (src/rglob, .py only) | Code LOC | Total Lines |", + "|------------------------------|----------|-------------|", + ] + for name, data in sorted_mods[:10]: + module_rows.append( + f"| {name} | {format_number(data['code'])} | {format_number(data['total'])} |" + ) + + md: list[str] = [ + "## Repository Stats", + "", + f"- Code LOC (approx): {format_number(overall['code'])}", + f"- Total lines (tracked files): {format_number(overall['total'])}", + "", + *pie_lines, + "", + "### LOC by Area", + *area_rows, + "", + "### Code LOC by Module (Python only)", + *module_pie_lines, + "", + "### Top Python Modules (src/rglob)", + *module_rows, + "", + "_Note: LOC approximates non-blank, non-comment lines. Module " + "breakdown counts only Python files._", + ] + return "\n".join(md) + + +def update_overview(new_section: str) -> None: + """Insert or append the generated stats section in ``OVERVIEW.md``.""" + begin = "" + end = "" + generated = f"{begin}\n\n{new_section}\n\n{end}\n" + + if not OVERVIEW_PATH.exists(): + OVERVIEW_PATH.write_text( + "# rglob — Repository Overview\n\n" + "Auto-generated repository stats live below; everything outside\n" + "the `REPO-STATS` markers is hand-written.\n\n" + generated, + encoding="utf-8", + ) + return + + content = OVERVIEW_PATH.read_text(encoding="utf-8") + if begin in content and end in content: + pre, _, rest = content.partition(begin) + _, _, post = rest.partition(end) + OVERVIEW_PATH.write_text(pre + generated + post, encoding="utf-8") + else: + OVERVIEW_PATH.write_text(content.rstrip() + "\n\n---\n\n" + generated, encoding="utf-8") + + +def main() -> None: + """Generate current stats and persist them to ``OVERVIEW.md``.""" + stats = aggregate_stats() + md = generate_markdown(stats) + update_overview(md) + print(f"Updated {OVERVIEW_PATH.relative_to(REPO_ROOT)} with repository stats section.") + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py deleted file mode 100644 index ca1eaa6..0000000 --- a/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -"""Package configuration for rglob.""" - -from setuptools import setup - -setup(name='rglob', - version='1.7', - description='Python Recursive Glob', - author='Christopher Piekarski', - author_email='chris@cpiekarski.com', - maintainer='Christopher Piekarski', - maintainer_email='chris@cpiekarski.com', - keywords=['glob','rglob','recursive','line counter'], - python_requires='>=3.5', - classifiers=[ - 'Intended Audience :: Developers', - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 3.5', - 'Operating System :: OS Independent', - ], - url='http://cpiekarski.com/2011/09/23/python-recursive-glob/', - packages=['rglob'], - provides=['rglob'], - license='OSI Approved Apache Software License', - entry_points={ - 'console_scripts': [ - 'rglob = rglob.cli:main', - ], - }, - ) diff --git a/src/rglob/__init__.py b/src/rglob/__init__.py new file mode 100644 index 0000000..36f96ef --- /dev/null +++ b/src/rglob/__init__.py @@ -0,0 +1,43 @@ +"""Public API for the `rglob` package. + +The 2.0 surface exposes: + +- :func:`find` / :func:`find_all` — the modern Path-yielding API +- :func:`rglob` / :func:`rglob_` — legacy wrappers that now return + ``list[Path]`` at 2.0 (see ADR-0003) +- :func:`lcount` / :func:`tsize` — line-count and size aggregation helpers +- :func:`kilobytes` / :func:`megabytes` / :func:`gigabytes` / :func:`terabytes` + — binary-prefix conversion helpers + +The agent-facing API lives in :mod:`rglob.agent` (SemVer-locked, see +ADR-0009) and the CLI lives in :mod:`rglob.cli`. +""" + +from rglob.rglob import ( + find, + find_all, + gigabytes, + kilobytes, + lcount, + megabytes, + rglob, + rglob_, + terabytes, + tsize, +) + +__version__ = "2.0.0" + +__all__ = [ + "__version__", + "find", + "find_all", + "gigabytes", + "kilobytes", + "lcount", + "megabytes", + "rglob", + "rglob_", + "terabytes", + "tsize", +] diff --git a/src/rglob/_count.py b/src/rglob/_count.py new file mode 100644 index 0000000..f2a9f77 --- /dev/null +++ b/src/rglob/_count.py @@ -0,0 +1,128 @@ +"""Structured count helpers for agent-aware commands.""" + +from pathlib import Path + +from rglob import find as _find +from rglob.agent._models import CountOptions, ErrorCode, ErrorInfo, Stats +from rglob.agent._runtime import ( + _absolute, + _is_contained, + _strict_base_error, + check_timeout, + deadline_from_timeout, + timeout_error, +) + + +def _countable(line: str, opts: CountOptions) -> bool: + """Return true when a line should be counted.""" + stripped = line.strip() + if opts.no_empty and not stripped: + return False + return not (opts.no_comments and stripped.startswith("#")) + + +def _iter_count_files(opts: CountOptions) -> list[Path]: + """Return candidate files for count operations.""" + return list( + _find( + opts.base, + opts.patterns, + exclude=opts.exclude, + max_depth=opts.max_depth, + hidden=opts.hidden, + follow_symlinks=opts.follow_symlinks, + case_sensitive=opts.case_sensitive, + sort=opts.sort, + on_error="ignore", + kinds=opts.kinds, + min_size=opts.min_size, + max_size=opts.max_file_size or opts.max_size, + newer_than=opts.newer_than, + older_than=opts.older_than, + newer_than_file=opts.newer_than_file, + perm=opts.perm, + uid=opts.uid, + gid=opts.gid, + respect_gitignore=opts.respect_gitignore, + ) + ) + + +def count_all(opts: CountOptions) -> Stats: + """Count files, lines, and bytes using CountOptions.""" + files = _iter_count_files(opts) + errors: list[ErrorInfo] = [] + total_lines = 0 + total_bytes = 0 + truncated = False + truncated_reason: str | None = None + counted_files = 0 + deadline = deadline_from_timeout(opts.timeout_seconds) + + for path in files: + try: + check_timeout(deadline) + except TimeoutError: + truncated = True + truncated_reason = "timeout" + if opts.include_errors: + errors.append(timeout_error()) + break + + if opts.limit is not None and counted_files >= opts.limit: + truncated = True + truncated_reason = "limit" + break + + if opts.strict_base and not _is_contained(path, opts.base): + if opts.include_errors: + errors.append(_strict_base_error(path, _absolute(opts.base))) + continue + + try: + file_size = path.stat().st_size + except OSError as exc: + if opts.include_errors: + errors.append(ErrorInfo(ErrorCode.UNREADABLE, str(exc), _absolute(path))) + continue + + if opts.max_bytes is not None and total_bytes + file_size > opts.max_bytes: + truncated = True + truncated_reason = "max_bytes" + break + + try: + data = path.read_bytes() + except OSError as exc: + if opts.include_errors: + errors.append(ErrorInfo(ErrorCode.UNREADABLE, str(exc), _absolute(path))) + continue + + lines = data.decode(opts.encoding, errors="replace").splitlines() + for line in lines: + try: + check_timeout(deadline) + except TimeoutError: + truncated = True + truncated_reason = "timeout" + if opts.include_errors: + errors.append(timeout_error(_absolute(path))) + break + if _countable(line, opts): + total_lines += 1 + total_bytes += len(data) + counted_files += 1 + if truncated: + break + + return Stats( + files=counted_files, + lines=total_lines, + bytes=total_bytes, + total_files_searched=len(files), + bytes_read=total_bytes, + errors=errors, + truncated=truncated, + truncated_reason=truncated_reason, + ) diff --git a/src/rglob/_dupes.py b/src/rglob/_dupes.py new file mode 100644 index 0000000..a93d359 --- /dev/null +++ b/src/rglob/_dupes.py @@ -0,0 +1,123 @@ +"""Duplicate-file detection used by the `rglob dupes` subcommand. + +Strategy: +1. Bucket candidate files by exact size. +2. For each size bucket with ≥2 entries, bucket again by the hash of the + first 4 KiB. +3. For each surviving bucket with ≥2 entries, hash the full contents. + +This three-stage approach minimises I/O — large unique files only ever read +their first 4 KiB. + +Uses :func:`xxhash.xxh3_64` when available (under the `[ext]` extra) and +falls back to :func:`hashlib.file_digest` with BLAKE2b otherwise — both +are fast enough in 2026 that the choice is rarely visible. +""" + +from __future__ import annotations + +import hashlib +from collections import defaultdict +from collections.abc import Callable, Iterable +from pathlib import Path + +TimeoutCheck = Callable[[], None] + + +def _fast_hash(data: bytes) -> str: + """Return a fast 64-bit hex digest of `data`. + + Uses :func:`xxhash.xxh3_64` when the optional ``xxhash`` dependency is + installed (via ``pip install rglob[ext]``); falls back to the stdlib + BLAKE2b otherwise — both are fast enough in 2026 that the choice is + rarely visible. + """ + try: + import xxhash + + return xxhash.xxh3_64(data).hexdigest() + except ImportError: # pragma: no cover - exercised only when xxhash absent + return hashlib.blake2b(data, digest_size=16).hexdigest() + + +# Chunk read for the fast-path "first 4 KiB" bucket. +_HEAD_BYTES = 4 * 1024 + + +def _check_timeout(timeout_check: TimeoutCheck | None) -> None: + """Run the duplicate timeout hook when one was supplied.""" + if timeout_check is not None: + timeout_check() + + +def _hash_head(path: Path, *, timeout_check: TimeoutCheck | None = None) -> str: + """Hash the first 4 KiB of a file.""" + _check_timeout(timeout_check) + with path.open("rb") as fp: + chunk = fp.read(_HEAD_BYTES) + _check_timeout(timeout_check) + return _fast_hash(chunk) + + +def _hash_full(path: Path, *, timeout_check: TimeoutCheck | None = None) -> str: + """Hash the entire file in 64 KiB chunks.""" + hasher_state = hashlib.blake2b(digest_size=16) + with path.open("rb") as fp: + while True: + _check_timeout(timeout_check) + chunk = fp.read(64 * 1024) + if not chunk: + break + hasher_state.update(chunk) + _check_timeout(timeout_check) + return hasher_state.hexdigest() + + +def find_duplicates( + paths: Iterable[Path], + *, + timeout_check: TimeoutCheck | None = None, +) -> list[list[Path]]: + """Group paths by content equivalence. + + Returns a list of groups, where each group contains 2+ paths sharing + identical bytes. Files that are unique are *not* returned (only + duplicate groups). + """ + by_size: dict[int, list[Path]] = defaultdict(list) + for path in paths: + try: + _check_timeout(timeout_check) + if not path.is_file(): + continue + by_size[path.stat().st_size].append(path) + except OSError: # pragma: no cover - rare + continue + + candidates: list[list[Path]] = [] + for group in by_size.values(): + if len(group) < 2: + continue + by_head: dict[str, list[Path]] = defaultdict(list) + for path in group: + try: + _check_timeout(timeout_check) + by_head[_hash_head(path, timeout_check=timeout_check)].append(path) + except OSError: # pragma: no cover - rare + continue + candidates.extend(head_group for head_group in by_head.values() if len(head_group) >= 2) + + final_groups: list[list[Path]] = [] + for group in candidates: + by_full: dict[str, list[Path]] = defaultdict(list) + for path in group: + try: + _check_timeout(timeout_check) + by_full[_hash_full(path, timeout_check=timeout_check)].append(path) + except OSError: # pragma: no cover - rare + continue + final_groups.extend( + sorted(full_group) for full_group in by_full.values() if len(full_group) >= 2 + ) + + return final_groups diff --git a/src/rglob/_filters.py b/src/rglob/_filters.py new file mode 100644 index 0000000..9aca7f0 --- /dev/null +++ b/src/rglob/_filters.py @@ -0,0 +1,362 @@ +"""Filter parsers and predicates for the Phase 5 fun-features pass. + +This module is internal-but-stable: external callers should reach it through +:func:`rglob.find` rather than importing helpers directly. +""" + +from __future__ import annotations + +import os +import re +import stat +from collections.abc import Callable, Iterable +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Literal + +Kind = Literal["f", "d", "l", "x"] +PermMode = Literal["exact", "all", "any"] + +# ─── Size strings ───────────────────────────────────────────────────────────── + +_SIZE_RE = re.compile(r"^\s*([0-9]+(?:\.[0-9]+)?)\s*([kmgtKMGT]?)([iI]?)[bB]?\s*$") + +_UNIT_TO_BYTES: dict[str, int] = { + "": 1, + "K": 2**10, + "M": 2**20, + "G": 2**30, + "T": 2**40, +} + + +def parse_size(value: int | float | str) -> int: + """Parse a size into bytes. + + Accepts plain numbers (bytes) or strings like ``"1K"``, ``"2.5M"``, + ``"3GiB"``, ``"100 KB"``. Binary prefixes are assumed (KiB == 1024 B); + the optional ``i``/``B`` suffixes are ignored. + + Raises: + ValueError: if the string can't be parsed. + """ + if isinstance(value, (int, float)): + return int(value) + match = _SIZE_RE.match(value) + if not match: + raise ValueError(f"unparseable size: {value!r}") + number, unit, _i = match.groups() + return int(float(number) * _UNIT_TO_BYTES[unit.upper()]) + + +# ─── Time strings ───────────────────────────────────────────────────────────── + +_DURATION_RE = re.compile(r"^\s*([0-9]+(?:\.[0-9]+)?)\s*([smhdwSMHDW])\s*$") + +_DURATION_UNIT_SECONDS: dict[str, float] = { + "s": 1.0, + "m": 60.0, + "h": 3600.0, + "d": 86400.0, + "w": 604800.0, +} + + +def parse_time(value: datetime | timedelta | str) -> datetime: + """Parse a time expression into a tz-aware :class:`datetime`. + + Accepts: + - a :class:`datetime` (returned as-is — naive datetimes get UTC). + - a :class:`timedelta` (interpreted as "now minus delta"). + - an ISO-8601-style date string (``"2026-05-14"`` or + ``"2026-05-14T12:00:00"``). + - a relative duration like ``"7d"``, ``"3h"``, ``"2w"`` + (interpreted as "now minus that"). + """ + if isinstance(value, datetime): + return value if value.tzinfo else value.replace(tzinfo=UTC) + now = datetime.now(UTC) + if isinstance(value, timedelta): + return now - value + match = _DURATION_RE.match(value) + if match: + number, unit = match.groups() + return now - timedelta(seconds=float(number) * _DURATION_UNIT_SECONDS[unit.lower()]) + # Fall back to ISO parser. + try: + parsed = datetime.fromisoformat(value) + except ValueError as exc: + raise ValueError(f"unparseable time: {value!r}") from exc + return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC) + + +# ─── Kind predicates ────────────────────────────────────────────────────────── + + +def _entry_kind(entry: os.DirEntry[str]) -> set[Kind]: + """Return the kind tags applicable to a DirEntry.""" + tags: set[Kind] = set() + try: + if entry.is_symlink(): + tags.add("l") + except OSError: # pragma: no cover - is_symlink is documented not to raise + pass + try: + if entry.is_file(follow_symlinks=False): + tags.add("f") + except OSError: # pragma: no cover - rare + pass + try: + if entry.is_dir(follow_symlinks=False): + tags.add("d") + except OSError: # pragma: no cover - rare + pass + try: + st = entry.stat(follow_symlinks=False) + if st.st_mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH): + tags.add("x") + except OSError: # pragma: no cover - rare + pass + return tags + + +def kinds_match(entry: os.DirEntry[str], wanted: Iterable[Kind]) -> bool: + """Return ``True`` if the entry matches any wanted kind. + + Kept for backward-compatibility with direct callers. The walker + instead uses :func:`kinds_predicate` which returns a specialised + closure that avoids the generic set-intersection path (and the + `stat()` call that the executable check requires) for the common + single-kind cases. + """ + wanted_set = set(wanted) + if not wanted_set: + return True + return bool(_entry_kind(entry) & wanted_set) + + +def _entry_is_file(entry: os.DirEntry[str]) -> bool: + """Single-kind predicate: regular file (no symlink follow).""" + try: + return entry.is_file(follow_symlinks=False) + except OSError: # pragma: no cover - rare + return False + + +def _entry_is_dir(entry: os.DirEntry[str]) -> bool: + """Single-kind predicate: directory (no symlink follow).""" + try: + return entry.is_dir(follow_symlinks=False) + except OSError: # pragma: no cover - rare + return False + + +def _entry_is_symlink(entry: os.DirEntry[str]) -> bool: + """Single-kind predicate: symlink (any target type).""" + try: + return entry.is_symlink() + except OSError: # pragma: no cover - rare + return False + + +_EXEC_BITS = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + + +def _entry_is_executable(entry: os.DirEntry[str]) -> bool: + """Single-kind predicate: any exec bit set.""" + try: + return bool(entry.stat(follow_symlinks=False).st_mode & _EXEC_BITS) + except OSError: # pragma: no cover - rare + return False + + +_SINGLE_KIND_PREDICATES: dict[Kind, Callable[[os.DirEntry[str]], bool]] = { + "f": _entry_is_file, + "d": _entry_is_dir, + "l": _entry_is_symlink, + "x": _entry_is_executable, +} + + +def kinds_predicate(wanted: Iterable[Kind]) -> Callable[[os.DirEntry[str]], bool]: + """Return an entry predicate specialised for the wanted kind set. + + The single-kind cases (``{"f"}``, ``{"d"}``, ``{"l"}``, ``{"x"}``) + skip the four syscalls + ``stat()`` that the generic union path + requires when only one kind matters — a measurable walker-time + win when an agent filters by file kind. + """ + wanted_set = frozenset(wanted) + if not wanted_set: + return lambda _entry: True + if len(wanted_set) == 1: + return _SINGLE_KIND_PREDICATES[next(iter(wanted_set))] + # Multi-kind: union semantics. Could be specialised further + # (e.g. `{"f","x"}` is just exec-bit-on regular files) but the + # gain shrinks as the set grows; keep the general path. + return lambda entry: bool(_entry_kind(entry) & wanted_set) + + +# ─── Size / mtime predicates ────────────────────────────────────────────────── + + +def size_predicate( + min_size: int | float | str | None, + max_size: int | float | str | None, +) -> Callable[[os.DirEntry[str]], bool]: + """Build a predicate that checks an entry's size against min/max bounds.""" + lo = parse_size(min_size) if min_size is not None else None + hi = parse_size(max_size) if max_size is not None else None + if lo is None and hi is None: + return lambda _entry: True + + def _check(entry: os.DirEntry[str]) -> bool: + try: + if not entry.is_file(follow_symlinks=False): + # Non-files (dirs, symlinks) always pass size filters. + return True + size = entry.stat(follow_symlinks=False).st_size + except OSError: # pragma: no cover - rare + return True + if lo is not None and size < lo: + return False + return not (hi is not None and size > hi) + + return _check + + +def mtime_predicate( + newer_than: datetime | timedelta | str | None, + older_than: datetime | timedelta | str | None, +) -> Callable[[os.DirEntry[str]], bool]: + """Build a predicate that checks an entry's mtime against bounds.""" + after = parse_time(newer_than) if newer_than is not None else None + before = parse_time(older_than) if older_than is not None else None + if after is None and before is None: + return lambda _entry: True + + def _check(entry: os.DirEntry[str]) -> bool: + try: + mtime = entry.stat(follow_symlinks=False).st_mtime + except OSError: # pragma: no cover - rare + return True + ts = datetime.fromtimestamp(mtime, tz=UTC) + if after is not None and ts < after: + return False + return not (before is not None and ts > before) + + return _check + + +# ─── find(1)-style predicates ──────────────────────────────────────────────── + + +def parse_perm(value: int | str) -> tuple[int, PermMode]: + """Parse a find-style permission expression. + + Plain values (``"644"``, ``"0o644"``, ``420``) require an exact mode + match. A leading ``-`` requires all mask bits; a leading ``/`` requires + any mask bit. + """ + if isinstance(value, int): + return value, "exact" + + text = value.strip() + mode: PermMode = "exact" + if text.startswith("-"): + mode = "all" + text = text[1:] + elif text.startswith("/"): + mode = "any" + text = text[1:] + + try: + parsed = int(text, 8) + except ValueError as exc: + raise ValueError(f"unparseable permission mode: {value!r}") from exc + return parsed, mode + + +def perm_predicate(perm: int | str | None) -> Callable[[os.DirEntry[str]], bool]: + """Build a predicate that checks POSIX permission bits.""" + if perm is None: + return lambda _entry: True + + mask, mode = parse_perm(perm) + + def _check(entry: os.DirEntry[str]) -> bool: + try: + bits = stat.S_IMODE(entry.stat(follow_symlinks=False).st_mode) + except OSError: # pragma: no cover - rare + return True + if mode == "all": + return bits & mask == mask + if mode == "any": + return bool(bits & mask) + return bits == mask + + return _check + + +def owner_predicate( + uid: int | None, + gid: int | None, +) -> Callable[[os.DirEntry[str]], bool]: + """Build a predicate that checks POSIX uid/gid ownership.""" + if uid is None and gid is None: + return lambda _entry: True + if os.name != "posix": # pragma: no cover - exercised by non-POSIX CI only + raise ValueError("uid/gid filters are POSIX-only") + + def _check(entry: os.DirEntry[str]) -> bool: + try: + entry_stat = entry.stat(follow_symlinks=False) + except OSError: # pragma: no cover - rare + return True + if uid is not None and entry_stat.st_uid != uid: + return False + return not (gid is not None and entry_stat.st_gid != gid) + + return _check + + +# ─── .gitignore awareness ───────────────────────────────────────────────────── + + +def gitignore_matcher(base: Path) -> Callable[[Path], bool] | None: + """Return a "should ignore" predicate honouring `.gitignore` files. + + Walks `base` collecting every `.gitignore` (top-down) and merges them + into a per-tree :class:`pathspec.PathSpec`. Returns ``None`` if + :mod:`pathspec` isn't installed — callers should treat that as + "feature disabled" and fall back to the no-gitignore walk. + """ + try: + import pathspec + except ImportError: # pragma: no cover - optional extra + return None + + patterns: list[str] = [] + for gi in base.rglob(".gitignore"): + try: + patterns.extend(gi.read_text(encoding="utf-8").splitlines()) + except OSError: # pragma: no cover - unreadable .gitignore + continue + # pathspec >=1.0 renamed the pattern dialect from "gitwildmatch" to + # "gitignore"; fall back to the older name on installations that haven't + # bumped yet. Typed as `Any` so the union of generic PathSpec[...] types + # doesn't trip mypy --strict. + spec: object + try: + spec = pathspec.PathSpec.from_lines("gitignore", patterns) + except (ValueError, LookupError): # pragma: no cover - older pathspec + spec = pathspec.PathSpec.from_lines("gitwildmatch", patterns) + + def _should_ignore(path: Path) -> bool: + try: + rel = path.relative_to(base) + except ValueError: # pragma: no cover - paths are always under base + return False + return bool(spec.match_file(rel.as_posix())) + + return _should_ignore diff --git a/src/rglob/_grep.py b/src/rglob/_grep.py new file mode 100644 index 0000000..13c839f --- /dev/null +++ b/src/rglob/_grep.py @@ -0,0 +1,422 @@ +r"""Content search helpers for the agent-aware grep command. + +The walker yields candidate files via :func:`rglob.find`. Each file is +streamed through a binary peek (for ``\x00`` detection) followed by an +``io.TextIOWrapper`` for line-by-line iteration — this avoids reading +the full file into memory and lets the loop break out as soon as +``--limit`` / ``--max-count`` / ``--max-bytes`` / ``--timeout`` fires, +which matters most on large files where only the first few lines need +to be searched. + +Trailing context (``--after`` / ``--context``) requires lookahead, so +matches that still need after-lines park in a small *pending* list. +Each subsequent line either feeds existing pending matches or starts a +new one; pending matches finalize when their counter hits zero or the +file ends. +""" + +import io +import re +from collections import deque +from collections.abc import Iterable +from dataclasses import dataclass, field +from pathlib import Path + +from rglob import find as _find +from rglob.agent._models import ErrorCode, ErrorInfo, GrepOptions, LineMatch, LineSearchResult +from rglob.agent._runtime import ( + _absolute, + _is_contained, + _strict_base_error, + check_timeout, + deadline_from_timeout, + timeout_error, +) + +# Bytes peeked from each file to classify it as binary before streaming. +_BINARY_PROBE_BYTES = 1024 + +# Files at or below this size are read with a single `fp.read()` and +# split via `str.splitlines()` rather than wrapped in +# :class:`io.TextIOWrapper`. The wrapper has measurable per-file +# instantiation overhead that dominates for small files where +# early-termination isn't possible anyway. Large files keep the +# streaming path so `--limit` / `--max-count` / `--timeout` short- +# circuit reading. +_STREAM_THRESHOLD_BYTES = 64 * 1024 + + +def _compile_pattern(opts: GrepOptions) -> re.Pattern[str]: + """Compile a grep pattern according to options.""" + pattern = re.escape(opts.pattern) if opts.fixed_string else opts.pattern + if opts.word: + pattern = rf"\b(?:{pattern})\b" + flags = re.IGNORECASE if opts.ignore_case else 0 + return re.compile(pattern, flags) + + +def _is_binary(data: bytes) -> bool: + """Return true when a byte sample looks binary.""" + return b"\x00" in data[:_BINARY_PROBE_BYTES] + + +def _iter_candidate_files(opts: GrepOptions) -> Iterable[Path]: + """Yield files selected by the path side of GrepOptions.""" + return _find( + opts.base, + opts.paths, + exclude=opts.exclude, + max_depth=opts.max_depth, + hidden=opts.hidden, + follow_symlinks=opts.follow_symlinks, + case_sensitive=opts.case_sensitive, + sort=opts.sort, + on_error="ignore", + kinds=opts.kinds, + min_size=opts.min_size, + max_size=opts.max_file_size or opts.max_size, + newer_than=opts.newer_than, + older_than=opts.older_than, + newer_than_file=opts.newer_than_file, + perm=opts.perm, + uid=opts.uid, + gid=opts.gid, + respect_gitignore=opts.respect_gitignore, + ) + + +@dataclass +class _PendingMatch: + """In-flight match whose `after` context is still being collected.""" + + line_match: LineMatch + remaining_after: int + after: list[str] = field(default_factory=list) + + def feed(self, line: str) -> bool: + """Append `line` to this match's after context. Returns True when full.""" + self.after.append(line) + self.remaining_after -= 1 + return self.remaining_after == 0 + + def finalize(self) -> LineMatch: + """Materialise the LineMatch with whatever after lines we collected.""" + return LineMatch( + path=self.line_match.path, + line_number=self.line_match.line_number, + content=self.line_match.content, + before=self.line_match.before, + after=self.after, + encoding=self.line_match.encoding, + ) + + +def grep_iter(opts: GrepOptions) -> Iterable[LineMatch]: + """Yield each :class:`LineMatch` as the search progresses. + + Useful for ``--jsonl`` consumers that want one record per line of + output. Errors / truncation / per-file stats are *not* surfaced + through this generator — the caller can follow up with + :func:`grep_all` against the same options to get a full + :class:`LineSearchResult` summary. + + The current implementation buffers internally via :func:`grep_all` + and yields after the search completes; the public contract still + holds (one ``LineMatch`` per yielded value, in find order), and a + future refactor can make the underlying scan truly lazy without + changing this signature. + """ + yield from grep_all(opts).results + + +def grep_all(opts: GrepOptions) -> LineSearchResult: + """Search files selected by GrepOptions and return structured matches. + + Streams each file line-by-line via :class:`io.TextIOWrapper` so the + walker stops reading as soon as ``--limit`` / ``--max-count`` / + ``--max-bytes`` / ``--timeout`` fires. Trailing-context matches are + parked in a small pending list and finalised when their counter + reaches zero or the file ends. + """ + try: + regex = _compile_pattern(opts) + except re.error as exc: + return LineSearchResult( + results=[], + truncated=False, + total_files_searched=0, + bytes_read=0, + errors=[ErrorInfo(ErrorCode.REGEX, str(exc), None)], + truncated_reason=None, + ) + + results: list[LineMatch] = [] + errors: list[ErrorInfo] = [] + total_files = 0 + bytes_read = 0 + truncated = False + truncated_reason: str | None = None + files_with_matches = opts.files_with_matches + count_only = opts.count_only + summary_only = files_with_matches or count_only + # `--files-with-matches` and `--count-only` emit one summary + # `LineMatch` per matching file; per-line context never makes sense + # for them, so collapse the cold path and skip per-line `before`/ + # `after` tracking entirely. + before_count = 0 if summary_only else (opts.context if opts.context else opts.before) + after_count = 0 if summary_only else (opts.context if opts.context else opts.after) + deadline = deadline_from_timeout(opts.timeout_seconds) + limit = opts.limit + max_count = opts.max_count + invert = opts.invert + encoding = opts.encoding + include_errors = opts.include_errors + regex_search = regex.search # bound-method hoist for the hot loop + + pending: list[_PendingMatch] = [] + + for path in _iter_candidate_files(opts): + try: + check_timeout(deadline) + except TimeoutError: + truncated = True + truncated_reason = "timeout" + if opts.include_errors: + errors.append(timeout_error()) + break + + if opts.strict_base and not _is_contained(path, opts.base): + if opts.include_errors: + errors.append(_strict_base_error(path, _absolute(opts.base))) + continue + + try: + file_size = path.stat().st_size + except OSError as exc: + if opts.include_errors: + errors.append(ErrorInfo(ErrorCode.UNREADABLE, str(exc), _absolute(path))) + continue + + if opts.max_bytes is not None and bytes_read + file_size > opts.max_bytes: + truncated = True + truncated_reason = "max_bytes" + break + + # `bytes_read` is a per-file budget account — adding `file_size` + # up front (rather than tracking actual bytes consumed by + # streaming) keeps the `--max-bytes` gate stable and pessimistic + # in the right direction (we never over-budget). + bytes_read += file_size + total_files += 1 + file_truncated = False + file_path_abs = _absolute(path) + + try: + with path.open("rb") as fp: + # Choose between read-all and streaming. Small files pay + # too much for `TextIOWrapper` instantiation; large files + # benefit from the early-break short-circuit it enables. + if file_size <= _STREAM_THRESHOLD_BYTES: + data = fp.read() + if _is_binary(data) and not opts.text: + if opts.include_errors: + errors.append( + ErrorInfo(ErrorCode.BINARY, "binary file skipped", file_path_abs) + ) + continue + text = data.decode(opts.encoding, errors="replace") + line_iter: Iterable[str] = iter(text.splitlines()) + text_stream: io.TextIOWrapper | None = None + else: + head = fp.read(_BINARY_PROBE_BYTES) + if _is_binary(head) and not opts.text: + if opts.include_errors: + errors.append( + ErrorInfo(ErrorCode.BINARY, "binary file skipped", file_path_abs) + ) + continue + fp.seek(0) + # `TextIOWrapper` takes ownership of `fp` for line + # iteration; the outer `with` still closes the file. + text_stream = io.TextIOWrapper(fp, encoding=opts.encoding, errors="replace") + line_iter = text_stream + + previous: deque[str] = deque(maxlen=before_count) + pending.clear() + file_match_counter = 0 + + try: + if after_count == 0: + # Hot path: no after-context, no pending list. + # `committed = len(results)` (no pending entries + # ever exist on this branch), so `--limit` and + # `--max-count` are dirt-cheap O(1) checks. + for line_number, raw_line in enumerate(line_iter, start=1): + line = raw_line.rstrip("\r\n") + + try: + check_timeout(deadline) + except TimeoutError: + truncated = True + truncated_reason = "timeout" + if include_errors: + errors.append(timeout_error(file_path_abs)) + file_truncated = True + break + + matched = regex_search(line) is not None + if invert: + matched = not matched + + if matched: + if files_with_matches: + # One stub per matching file → stop reading + # *this* file. `file_truncated` only flips + # if the global limit is hit, so the outer + # file loop continues to the next file. + results.append( + LineMatch( + path=file_path_abs, + line_number=0, + content="", + before=[], + after=[], + encoding=encoding, + ) + ) + if limit is not None and len(results) >= limit: + truncated = True + truncated_reason = "limit" + file_truncated = True + break + if count_only: + file_match_counter += 1 + continue + results.append( + LineMatch( + path=file_path_abs, + line_number=line_number, + content=line, + before=list(previous), + after=[], + encoding=encoding, + ) + ) + committed = len(results) + if limit is not None and committed >= limit: + truncated = True + truncated_reason = "limit" + file_truncated = True + break + if max_count is not None and committed >= max_count: + truncated = True + truncated_reason = "max_count" + file_truncated = True + break + + previous.append(line) + + # `count_only` defers emission until the file is + # fully scanned (or short-circuited by a limit on + # a per-file basis — N/A here since count_only + # never breaks early on its own). + if count_only and file_match_counter > 0: + results.append( + LineMatch( + path=file_path_abs, + line_number=file_match_counter, + content="", + before=[], + after=[], + encoding=encoding, + ) + ) + if limit is not None and len(results) >= limit: + truncated = True + truncated_reason = "limit" + file_truncated = True + else: + # Cold path: trailing context (`--after`/`--context`). + # Pending matches park here until their after-counter + # hits zero or the file ends. + for line_number, raw_line in enumerate(line_iter, start=1): + line = raw_line.rstrip("\r\n") + + if pending: + still_pending: list[_PendingMatch] = [] + for pmatch in pending: + if pmatch.feed(line): + results.append(pmatch.finalize()) + else: + still_pending.append(pmatch) + pending = still_pending + + try: + check_timeout(deadline) + except TimeoutError: + truncated = True + truncated_reason = "timeout" + if include_errors: + errors.append(timeout_error(file_path_abs)) + file_truncated = True + break + + matched = regex_search(line) is not None + if invert: + matched = not matched + + committed = len(results) + len(pending) + if matched and (limit is None or committed < limit): + pending.append( + _PendingMatch( + LineMatch( + path=file_path_abs, + line_number=line_number, + content=line, + before=list(previous), + after=[], + encoding=encoding, + ), + after_count, + ) + ) + committed = len(results) + len(pending) + + if limit is not None and committed >= limit and not pending: + truncated = True + truncated_reason = "limit" + file_truncated = True + break + if max_count is not None and committed >= max_count and not pending: + truncated = True + truncated_reason = "max_count" + file_truncated = True + break + + previous.append(line) + finally: + # Flush pending matches with whatever after-context we + # collected. Pending is per-file: matches never span + # files. `text_stream.detach()` releases the underlying + # `fp` so the outer `with` can close it; the read-all + # path has no wrapper to detach. + results.extend(p.finalize() for p in pending) + pending.clear() + if text_stream is not None: + text_stream.detach() + except OSError as exc: + if opts.include_errors: + errors.append(ErrorInfo(ErrorCode.UNREADABLE, str(exc), file_path_abs)) + continue + + if file_truncated: + break + + return LineSearchResult( + results=results, + truncated=truncated, + total_files_searched=total_files, + bytes_read=bytes_read, + errors=errors, + truncated_reason=truncated_reason, + ) diff --git a/src/rglob/agent/__init__.py b/src/rglob/agent/__init__.py new file mode 100644 index 0000000..38aeb04 --- /dev/null +++ b/src/rglob/agent/__init__.py @@ -0,0 +1,253 @@ +"""Stable agent-facing API for rglob.""" + +from collections.abc import Iterator +from pathlib import Path + +from rglob.agent._introspection import all_schemas, schema_for +from rglob.agent._models import ( + AGENT_API_VERSION, + SCHEMA_VERSION, + CapabilityReport, + CountOptions, + Duplicate, + DuplicateSearchResult, + ErrorCode, + ErrorInfo, + FileMatch, + FileSearchResult, + GrepOptions, + LineMatch, + LineSearchResult, + Stats, + WalkOptions, + error_envelope, + to_json_dict, +) + +__agent_api_version__ = AGENT_API_VERSION + +__all__ = [ + "SCHEMA_VERSION", + "CapabilityReport", + "CountOptions", + "Duplicate", + "DuplicateSearchResult", + "ErrorCode", + "ErrorInfo", + "FileMatch", + "FileSearchResult", + "GrepOptions", + "LineMatch", + "LineSearchResult", + "Stats", + "WalkOptions", + "__agent_api_version__", + "all_schemas", + "count", + "error_envelope", + "find_duplicates", + "grep", + "grep_all", + "schema_for", + "search", + "search_all", + "to_json_dict", +] + + +def _error_from_exception(exc: Exception, path: Path | None = None) -> ErrorInfo: + """Convert an exception into the public agent error model.""" + if isinstance(exc, ValueError): + code = ErrorCode.BAD_PREDICATE + elif isinstance(exc, PermissionError): + code = ErrorCode.PERM + else: + code = ErrorCode.UNREADABLE + return ErrorInfo(code=code, message=str(exc), path=path) + + +def _iter_paths(opts: WalkOptions, *, force_files: bool = False) -> Iterator[Path]: + """Yield filesystem paths selected by WalkOptions.""" + from rglob import find as _find + from rglob._filters import Kind + + kinds: list[Kind] = ["f"] if force_files else opts.kinds + return _find( + opts.base, + opts.patterns, + exclude=opts.exclude, + max_depth=opts.max_depth, + hidden=opts.hidden, + follow_symlinks=opts.follow_symlinks, + case_sensitive=opts.case_sensitive, + sort=opts.sort, + on_error="ignore", + kinds=kinds, + min_size=opts.min_size, + max_size=opts.max_file_size or opts.max_size, + newer_than=opts.newer_than, + older_than=opts.older_than, + newer_than_file=opts.newer_than_file, + perm=opts.perm, + uid=opts.uid, + gid=opts.gid, + respect_gitignore=opts.respect_gitignore, + ) + + +def search(opts: WalkOptions) -> Iterator[FileMatch]: + """Yield file matches for agent callers without raising operational errors.""" + return iter(search_all(opts).results) + + +def search_all(opts: WalkOptions) -> FileSearchResult: + """Return a non-raising structured file search result.""" + from rglob.agent._runtime import collect_file_search + + try: + return collect_file_search( + _iter_paths(opts), + base=opts.base, + limit=opts.limit, + max_bytes=opts.max_bytes, + timeout_seconds=opts.timeout_seconds, + strict_base=opts.strict_base, + include_errors=opts.include_errors, + ) + except Exception as exc: + return FileSearchResult( + results=[], + truncated=False, + total_files_searched=0, + bytes_read=0, + errors=[_error_from_exception(exc)], + truncated_reason=None, + ) + + +def grep(opts: GrepOptions) -> Iterator[LineMatch]: + """Yield content matches for agent callers without raising operational errors.""" + return iter(grep_all(opts).results) + + +def grep_all(opts: GrepOptions) -> LineSearchResult: + """Return a non-raising structured content search result.""" + from rglob._grep import grep_all as _grep_all + + try: + return _grep_all(opts) + except Exception as exc: + return LineSearchResult( + results=[], + truncated=False, + total_files_searched=0, + bytes_read=0, + errors=[_error_from_exception(exc)], + truncated_reason=None, + ) + + +def count(opts: CountOptions) -> Stats: + """Return non-raising structured file, line, and byte counts.""" + from rglob._count import count_all as _count_all + + try: + return _count_all(opts) + except Exception as exc: + return Stats( + files=0, + lines=0, + bytes=0, + total_files_searched=0, + bytes_read=0, + errors=[_error_from_exception(exc)], + truncated=False, + truncated_reason=None, + ) + + +def _duplicate_record(group: list[Path], base: Path) -> Duplicate: + """Convert duplicate paths into a Duplicate record.""" + from rglob.agent._runtime import _absolute, _relative_path + + absolute_base = _absolute(base) + absolute_paths = [_absolute(path) for path in group] + size = 0 + if group: + try: + size = group[0].stat().st_size + except OSError: + size = 0 + return Duplicate( + paths=absolute_paths, + relative_paths=[_relative_path(path, absolute_base) for path in absolute_paths], + size=size, + digest=None, + ) + + +def find_duplicates(opts: WalkOptions) -> DuplicateSearchResult: + """Return non-raising duplicate-file groups selected by WalkOptions.""" + from rglob._dupes import find_duplicates as _find_duplicate_groups + from rglob.agent._runtime import ( + _absolute, + _is_contained, + _strict_base_error, + check_timeout, + deadline_from_timeout, + timeout_error, + ) + + paths: list[Path] = [] + errors: list[ErrorInfo] = [] + try: + deadline = deadline_from_timeout(opts.timeout_seconds) + truncated = False + truncated_reason = None + + def check_deadline() -> None: + check_timeout(deadline) + + for path in _iter_paths(opts, force_files=True): + check_deadline() + if opts.strict_base and not _is_contained(path, opts.base): + if opts.include_errors: + errors.append(_strict_base_error(path, _absolute(opts.base))) + continue + paths.append(path) + groups = _find_duplicate_groups(paths, timeout_check=check_deadline) + check_deadline() + results = [_duplicate_record(group, opts.base) for group in groups] + bytes_read = sum(record.size * len(record.paths) for record in results) + if opts.limit is not None and len(results) > opts.limit: + results = results[: opts.limit] + truncated = True + truncated_reason = "limit" + return DuplicateSearchResult( + results=results, + truncated=truncated, + total_files_searched=len(paths), + bytes_read=bytes_read, + errors=errors, + truncated_reason=truncated_reason, + ) + except TimeoutError: + if opts.include_errors: + errors.append(timeout_error()) + return DuplicateSearchResult( + results=[], + truncated=True, + total_files_searched=len(paths), + bytes_read=0, + errors=errors, + truncated_reason="timeout", + ) + except Exception as exc: + return DuplicateSearchResult( + results=[], + truncated=False, + total_files_searched=0, + bytes_read=0, + errors=[_error_from_exception(exc)], + truncated_reason=None, + ) diff --git a/src/rglob/agent/_introspection.py b/src/rglob/agent/_introspection.py new file mode 100644 index 0000000..2b3451e --- /dev/null +++ b/src/rglob/agent/_introspection.py @@ -0,0 +1,322 @@ +"""Machine-readable command descriptions and capability reporting.""" + +import dataclasses +import importlib.util +import os +import types +from collections.abc import Callable +from datetime import datetime, timedelta +from enum import StrEnum +from pathlib import Path +from typing import Any, Literal, Union, cast, get_args, get_origin, get_type_hints + +import rglob +from rglob.agent import _models +from rglob.agent._models import ( + AGENT_API_VERSION, + SCHEMA_VERSION, + CapabilityReport, + ErrorCode, + ErrorInfo, + JsonValue, + PredicateStatus, + error_envelope, + to_json_dict, +) + +JSON_SCHEMA_VERSION = "https://json-schema.org/draft/2020-12/schema" +SCHEMA_BASE_ID = f"https://chris-piekarski.github.io/python-rglob/schemas/{SCHEMA_VERSION}" + +PUBLIC_TYPES: tuple[type[object], ...] = ( + _models.FileMatch, + _models.LineMatch, + _models.Stats, + _models.Duplicate, + _models.ErrorInfo, + _models.ErrorCode, + _models.FileSearchResult, + _models.LineSearchResult, + _models.DuplicateSearchResult, + _models.CapabilityReport, + _models.WalkOptions, + _models.GrepOptions, + _models.CountOptions, +) + +COMMAND_SCHEMAS: dict[str, tuple[str, str]] = { + "find": ("walk_options", "file_search_result"), + "grep": ("grep_options", "line_search_result"), + "count": ("count_options", "stats"), + "lcount": ("count_options", "stats"), + "tsize": ("count_options", "stats"), + "stats": ("count_options", "stats"), + "tree": ("walk_options", "file_search_result"), + "top": ("walk_options", "file_search_result"), + "dupes": ("walk_options", "duplicate_search_result"), +} + +COMMAND_ARGUMENTS: dict[str, list[dict[str, object]]] = { + "find": [{"name": "patterns", "type": "list[str]", "required": True}], + "grep": [{"name": "pattern", "type": "str", "required": True}], + "count": [{"name": "patterns", "type": "list[str]", "required": True}], + "lcount": [{"name": "pattern", "type": "str", "required": True}], + "tsize": [{"name": "pattern", "type": "str", "required": True}], + "stats": [{"name": "pattern", "type": "str", "required": False, "default": "*"}], + "tree": [{"name": "pattern", "type": "str", "required": False, "default": "*"}], + "top": [{"name": "pattern", "type": "str", "required": False, "default": "*"}], + "dupes": [{"name": "pattern", "type": "str", "required": False, "default": "*"}], +} + +COMMON_OPTIONS: list[dict[str, object]] = [ + {"name": "base", "type": "Path", "default": "."}, + {"name": "exclude", "type": "list[str]", "default": []}, + {"name": "max_depth", "type": "int | null", "default": None}, + {"name": "hidden", "type": "bool", "default": False}, + {"name": "follow_symlinks", "type": "bool", "default": False}, + {"name": "case_sensitive", "type": "bool | null", "default": None}, + {"name": "respect_gitignore", "type": "bool", "default": False}, + {"name": "limit", "type": "int | null", "default": None}, + {"name": "max_bytes", "type": "int | null", "default": None}, + {"name": "max_file_size", "type": "int | null", "default": None}, + {"name": "timeout_seconds", "type": "float | null", "default": None}, + {"name": "newer_than_file", "type": "Path | null", "default": None}, + {"name": "perm", "type": "int | str | null", "default": None}, + {"name": "uid", "type": "int | null", "default": None}, + {"name": "gid", "type": "int | null", "default": None}, +] + +COMMAND_SUMMARIES: dict[str, str] = { + "find": "List paths matching one or more glob patterns.", + "grep": "Search file contents for a regex or fixed string.", + "count": "Count matching files, lines, and bytes.", + "lcount": "Legacy line-count command.", + "tsize": "Legacy total-size command.", + "stats": "Summarise a glob result.", + "tree": "Render a tree of matching paths.", + "top": "Show largest matching files.", + "dupes": "Find duplicate file groups.", +} + + +def _snake_case(name: str) -> str: + """Convert a class name to a stable schema stem.""" + out: list[str] = [] + for index, char in enumerate(name): + if char.isupper() and index > 0: + out.append("_") + out.append(char.lower()) + return "".join(out) + + +def _is_union(origin: object) -> bool: + """Return true when a typing origin represents a union.""" + return origin in (Union, types.UnionType) + + +def _schema_for_literal(args: tuple[object, ...]) -> dict[str, object]: + """Build a schema for a Literal type.""" + return {"enum": sorted(args, key=str)} + + +def _schema_for_enum(enum_type: type[StrEnum]) -> dict[str, object]: + """Build a schema for a StrEnum.""" + return {"type": "string", "enum": [item.value for item in enum_type]} + + +def _schema_for_type(tp: object) -> dict[str, object]: + """Translate a Python annotation into a JSON Schema fragment.""" + origin = get_origin(tp) + args = get_args(tp) + + if origin is Literal: + return _schema_for_literal(args) + if _is_union(origin): + return {"anyOf": [_schema_for_type(arg) for arg in args]} + if origin is list: + item_type = args[0] if args else Any + return {"type": "array", "items": _schema_for_type(item_type)} + if origin is dict: + value_type = args[1] if len(args) == 2 else Any + return {"type": "object", "additionalProperties": _schema_for_type(value_type)} + + if tp is Any: + return {} + if tp is None or tp is type(None): + return {"type": "null"} + if tp is str: + return {"type": "string"} + if tp is int: + return {"type": "integer"} + if tp is float: + return {"type": "number"} + if tp is bool: + return {"type": "boolean"} + if tp is Path: + return {"type": "string", "format": "path"} + if tp is datetime: + return {"type": "string", "format": "date-time"} + if tp is timedelta: + return {"type": "string", "format": "duration"} + if isinstance(tp, type) and issubclass(tp, StrEnum): + return {"$ref": f"#/$defs/{tp.__name__}"} + if isinstance(tp, type) and dataclasses.is_dataclass(tp): + return {"$ref": f"#/$defs/{tp.__name__}"} + + if tp is object: + return {"type": "string"} + + raise NotImplementedError( + f"unhandled type in schema generator: {tp!r} (origin={origin}, args={args})" + ) + + +def _field_default(schema: dict[str, object], field: dataclasses.Field[object]) -> None: + """Attach a JSON-safe default to a property schema when one exists.""" + if field.default is not dataclasses.MISSING: + schema["default"] = to_json_dict(field.default) + return + + default_factory: Callable[[], object] | object = field.default_factory + if default_factory is dataclasses.MISSING: + return + schema["default"] = to_json_dict(cast("Callable[[], object]", default_factory)()) + + +def _schema_for_dataclass(cls: type[object]) -> dict[str, object]: + """Build an object schema for a dataclass.""" + hints = get_type_hints(cls, include_extras=True) + properties: dict[str, object] = {} + required: list[str] = [] + + for field in dataclasses.fields(cast("Any", cls)): + prop = _schema_for_type(hints[field.name]) + _field_default(prop, field) + properties[field.name] = prop + if field.default is dataclasses.MISSING and field.default_factory is dataclasses.MISSING: + required.append(field.name) + + schema: dict[str, object] = { + "type": "object", + "additionalProperties": False, + "properties": properties, + } + if required: + schema["required"] = required + return schema + + +def _definitions(root: type[object]) -> dict[str, object]: + """Return definitions needed by generated schemas.""" + defs: dict[str, object] = {"ErrorCode": _schema_for_enum(_models.ErrorCode)} + for typ in PUBLIC_TYPES: + if typ is root or typ is _models.ErrorCode: + continue + defs[typ.__name__] = _schema_for_dataclass(typ) + return defs + + +def _schema_for_public_type(typ: type[object]) -> dict[str, object]: + """Build the complete schema document for a public contract type.""" + schema_name = _snake_case(typ.__name__) + body = _schema_for_enum(typ) if issubclass(typ, StrEnum) else _schema_for_dataclass(typ) + + document: dict[str, object] = { + "$schema": JSON_SCHEMA_VERSION, + "$id": f"{SCHEMA_BASE_ID}/{schema_name}.json", + "title": typ.__name__, + "version": SCHEMA_VERSION, + **body, + } + document["$defs"] = _definitions(typ) + return document + + +def _schema_types() -> dict[str, type[object]]: + """Return public schema types keyed by stable stem.""" + return {_snake_case(typ.__name__): typ for typ in PUBLIC_TYPES} + + +def schema_for(name: str) -> dict[str, Any]: + """Generate a public JSON Schema document by stable stem.""" + try: + typ = _schema_types()[name] + except KeyError as exc: + raise ValueError(f"unknown schema: {name}") from exc + return cast("dict[str, Any]", _schema_for_public_type(typ)) + + +def all_schemas() -> dict[str, dict[str, Any]]: + """Generate every public JSON Schema document keyed by stable stem.""" + return {name: schema_for(name) for name in sorted(_schema_types())} + + +def command_schema(name: str) -> dict[str, Any]: + """Return input and output schemas for a command.""" + try: + input_name, output_name = COMMAND_SCHEMAS[name] + except KeyError as exc: + raise ValueError(f"unknown subcommand: {name}") from exc + return { + "input": schema_for(input_name), + "output": schema_for(output_name), + } + + +def describe_command(name: str) -> dict[str, Any]: + """Return a JSON-serializable command manifest.""" + schemas = command_schema(name) + return { + "name": name, + "summary": COMMAND_SUMMARIES[name], + "agent_api_version": AGENT_API_VERSION, + "schema_version": SCHEMA_VERSION, + "arguments": COMMAND_ARGUMENTS[name], + "options": COMMON_OPTIONS, + "schemas": schemas, + "default_limits": { + "limit": None, + "max_bytes": None, + "max_file_size": None, + "timeout_seconds": None, + }, + "supported_extras": list(capability_report().extras.keys()), + } + + +def capability_report() -> CapabilityReport: + """Report installed optional features and platform predicate support.""" + has_gitignore = importlib.util.find_spec("pathspec") is not None + has_ext = importlib.util.find_spec("xxhash") is not None + has_mcp = importlib.util.find_spec("mcp") is not None + posix_status: PredicateStatus = "POSIX-only" if os.name == "posix" else "unsupported" + return CapabilityReport( + agent_api_version=AGENT_API_VERSION, + schema_version=SCHEMA_VERSION, + package_version=rglob.__version__, + extras={"mcp": has_mcp, "gitignore": has_gitignore, "ext": has_ext}, + predicates={ + "perm": "supported", + "uid": posix_status, + "gid": posix_status, + "newer_than": "supported", + "older_than": "supported", + "respect_gitignore": "supported" if has_gitignore else "unsupported", + }, + mcp={"available": has_mcp, "transport": "stdio"}, + ) + + +def unknown_command_envelope(name: str) -> dict[str, JsonValue]: + """Return a stable error envelope for an unknown command name.""" + return error_envelope( + ErrorInfo( + code=ErrorCode.BAD_PREDICATE, + message=f"unknown subcommand: {name}", + path=None, + ) + ) + + +def json_ready(value: object) -> JsonValue: + """Return a JSON-ready representation for introspection payloads.""" + return to_json_dict(value) diff --git a/src/rglob/agent/_models.py b/src/rglob/agent/_models.py new file mode 100644 index 0000000..d8d46db --- /dev/null +++ b/src/rglob/agent/_models.py @@ -0,0 +1,269 @@ +"""Shared agent contract models and JSON serialization helpers.""" + +from dataclasses import dataclass, field, fields, is_dataclass +from datetime import UTC, datetime, timedelta +from enum import StrEnum +from pathlib import Path +from typing import Literal, TypeAlias + +from rglob._filters import Kind + +AGENT_API_VERSION = "1.0" +SCHEMA_VERSION = "1.0" + +JsonPrimitive: TypeAlias = str | int | float | bool | None +JsonValue: TypeAlias = JsonPrimitive | list[object] | dict[str, object] +PredicateStatus: TypeAlias = Literal["supported", "POSIX-only", "unsupported"] + + +class ErrorCode(StrEnum): + """Stable operational error codes for the v1 agent contract.""" + + PERM = "PERM" + UNREADABLE = "UNREADABLE" + BINARY = "BINARY" + TIMEOUT = "TIMEOUT" + REGEX = "REGEX" + BAD_PREDICATE = "BAD_PREDICATE" + UNSUPPORTED_PLATFORM = "UNSUPPORTED_PLATFORM" + + +@dataclass(frozen=True, slots=True) +class ErrorInfo: + """Machine-readable operational error.""" + + code: ErrorCode + message: str + path: Path | None + + +@dataclass(frozen=True, slots=True) +class FileMatch: + """A filesystem entry matched by an agent search.""" + + path: Path + relative_path: str + size: int + mtime: datetime + kinds: list[Kind] + errors: list[ErrorInfo] + + +@dataclass(frozen=True, slots=True) +class LineMatch: + """A content match returned by an agent grep operation.""" + + path: Path + line_number: int + content: str + before: list[str] + after: list[str] + encoding: str + + +@dataclass(frozen=True, slots=True) +class Duplicate: + """A duplicate-file group.""" + + paths: list[Path] + relative_paths: list[str] + size: int + digest: str | None + + +@dataclass(frozen=True, slots=True) +class Stats: + """Structured count result for files, lines, and bytes.""" + + files: int + lines: int + bytes: int + total_files_searched: int + bytes_read: int + errors: list[ErrorInfo] + truncated: bool + truncated_reason: str | None + + +@dataclass(frozen=True, slots=True) +class FileSearchResult: + """Concrete result wrapper for file searches.""" + + results: list[FileMatch] + truncated: bool + total_files_searched: int + bytes_read: int + errors: list[ErrorInfo] + truncated_reason: str | None + + +@dataclass(frozen=True, slots=True) +class LineSearchResult: + """Concrete result wrapper for content searches.""" + + results: list[LineMatch] + truncated: bool + total_files_searched: int + bytes_read: int + errors: list[ErrorInfo] + truncated_reason: str | None + + +@dataclass(frozen=True, slots=True) +class DuplicateSearchResult: + """Concrete result wrapper for duplicate searches.""" + + results: list[Duplicate] + truncated: bool + total_files_searched: int + bytes_read: int + errors: list[ErrorInfo] + truncated_reason: str | None + + +@dataclass(frozen=True, slots=True) +class CapabilityReport: + """Installed capabilities and version metadata.""" + + agent_api_version: str + schema_version: str + package_version: str + extras: dict[str, bool] + predicates: dict[str, PredicateStatus] + mcp: dict[str, str | bool] + + +@dataclass(frozen=True, slots=True) +class WalkOptions: + """Options for agent-facing filename searches.""" + + patterns: list[str] = field(default_factory=lambda: ["*"]) + base: Path = Path() + exclude: list[str] = field(default_factory=list) + max_depth: int | None = None + hidden: bool = False + kinds: list[Kind] = field(default_factory=list) + min_size: int | float | str | None = None + max_size: int | float | str | None = None + newer_than: datetime | timedelta | str | None = None + older_than: datetime | timedelta | str | None = None + newer_than_file: Path | None = None + perm: int | str | None = None + uid: int | None = None + gid: int | None = None + limit: int | None = None + max_bytes: int | None = None + max_file_size: int | None = None + timeout_seconds: float | None = None + strict_base: bool = True + follow_symlinks: bool = False + respect_gitignore: bool = False + include_errors: bool = True + case_sensitive: bool | None = None + sort: bool = True + + +@dataclass(frozen=True, slots=True) +class GrepOptions: + """Options for agent-facing content searches.""" + + pattern: str + paths: list[str] = field(default_factory=lambda: ["*"]) + base: Path = Path() + exclude: list[str] = field(default_factory=list) + max_depth: int | None = None + hidden: bool = False + kinds: list[Kind] = field(default_factory=lambda: ["f"]) + min_size: int | float | str | None = None + max_size: int | float | str | None = None + newer_than: datetime | timedelta | str | None = None + older_than: datetime | timedelta | str | None = None + newer_than_file: Path | None = None + perm: int | str | None = None + uid: int | None = None + gid: int | None = None + limit: int | None = None + max_bytes: int | None = None + max_file_size: int | None = None + timeout_seconds: float | None = None + strict_base: bool = True + follow_symlinks: bool = False + respect_gitignore: bool = False + include_errors: bool = True + case_sensitive: bool | None = None + sort: bool = True + fixed_string: bool = False + ignore_case: bool = False + context: int = 0 + before: int = 0 + after: int = 0 + max_count: int | None = None + word: bool = False + invert: bool = False + encoding: str = "utf-8" + text: bool = False + files_with_matches: bool = False + count_only: bool = False + + +@dataclass(frozen=True, slots=True) +class CountOptions: + """Options for agent-facing file, line, and byte counts.""" + + patterns: list[str] = field(default_factory=lambda: ["*"]) + base: Path = Path() + exclude: list[str] = field(default_factory=list) + max_depth: int | None = None + hidden: bool = False + kinds: list[Kind] = field(default_factory=lambda: ["f"]) + min_size: int | float | str | None = None + max_size: int | float | str | None = None + newer_than: datetime | timedelta | str | None = None + older_than: datetime | timedelta | str | None = None + newer_than_file: Path | None = None + perm: int | str | None = None + uid: int | None = None + gid: int | None = None + limit: int | None = None + max_bytes: int | None = None + max_file_size: int | None = None + timeout_seconds: float | None = None + strict_base: bool = True + follow_symlinks: bool = False + respect_gitignore: bool = False + include_errors: bool = True + case_sensitive: bool | None = None + sort: bool = True + no_empty: bool = False + no_comments: bool = False + encoding: str = "utf-8" + + +def _datetime_to_wire(value: datetime) -> str: + """Serialize a datetime as UTC ISO 8601 with a Z suffix.""" + aware = value if value.tzinfo is not None else value.replace(tzinfo=UTC) + return aware.astimezone(UTC).isoformat().replace("+00:00", "Z") + + +def to_json_dict(value: object) -> JsonValue: + """Convert agent dataclasses into JSON-safe built-in containers.""" + if is_dataclass(value) and not isinstance(value, type): + return {field.name: to_json_dict(getattr(value, field.name)) for field in fields(value)} + if isinstance(value, Path): + return str(value) + if isinstance(value, datetime): + return _datetime_to_wire(value) + if isinstance(value, StrEnum): + return str(value) + if isinstance(value, list | tuple): + return [to_json_dict(item) for item in value] + if isinstance(value, dict): + return {str(key): to_json_dict(item) for key, item in value.items()} + if value is None or isinstance(value, str | int | float | bool): + return value + return str(value) + + +def error_envelope(error: ErrorInfo) -> dict[str, JsonValue]: + """Return the stable agent error envelope.""" + return {"ok": False, "error": to_json_dict(error)} diff --git a/src/rglob/agent/_runtime.py b/src/rglob/agent/_runtime.py new file mode 100644 index 0000000..d75ac3b --- /dev/null +++ b/src/rglob/agent/_runtime.py @@ -0,0 +1,182 @@ +"""Internal helpers shared by agent-aware CLI and future public APIs.""" + +import json +import stat +import time +from collections.abc import Iterable +from datetime import UTC, datetime +from pathlib import Path + +from rglob._filters import Kind +from rglob.agent._models import ErrorCode, ErrorInfo, FileMatch, FileSearchResult, to_json_dict + + +def _absolute(path: Path) -> Path: + """Return an absolute normalized path without requiring the path to exist.""" + return path.expanduser().resolve(strict=False) + + +def _lexical_absolute(path: Path) -> Path: + """Return an absolute path without following symlink targets.""" + expanded = path.expanduser() + if expanded.is_absolute(): + return expanded + return Path.cwd() / expanded + + +def _relative_path(path: Path, base: Path) -> str: + """Return a POSIX-style path relative to base.""" + try: + return path.relative_to(base).as_posix() + except ValueError: + return path.name + + +def _error_from_oserror(exc: OSError, path: Path) -> ErrorInfo: + """Convert an OSError into a stable agent error.""" + code = ErrorCode.PERM if isinstance(exc, PermissionError) else ErrorCode.UNREADABLE + return ErrorInfo(code=code, message=str(exc), path=path) + + +def deadline_from_timeout(timeout_seconds: float | None) -> float | None: + """Return a monotonic deadline for a cooperative timeout.""" + if timeout_seconds is None: + return None + if timeout_seconds < 0.1: + raise ValueError("timeout_seconds must be at least 0.1") + return time.monotonic() + timeout_seconds + + +def check_timeout(deadline: float | None) -> None: + """Raise TimeoutError when a cooperative deadline expires.""" + if deadline is not None and time.monotonic() >= deadline: + raise TimeoutError("search timed out") + + +def timeout_error(path: Path | None = None) -> ErrorInfo: + """Build the stable timeout ErrorInfo.""" + return ErrorInfo(ErrorCode.TIMEOUT, "search timed out", path) + + +def _strict_base_error(path: Path, base: Path) -> ErrorInfo: + """Build an ErrorInfo for a strict-base containment violation.""" + return ErrorInfo( + ErrorCode.PERM, + f"path escapes strict base {base}", + _lexical_absolute(path), + ) + + +def _is_contained(path: Path, base: Path) -> bool: + """Return true when a resolved path is within the resolved base.""" + resolved_base = _absolute(base) + resolved_path = _absolute(path) + try: + resolved_path.relative_to(resolved_base) + except ValueError: + return False + return True + + +def _path_kinds(path: Path) -> list[Kind]: + """Return kind tags for a path.""" + kinds: list[Kind] = [] + try: + if path.is_symlink(): + kinds.append("l") + if path.is_file(): + kinds.append("f") + if path.is_dir(): + kinds.append("d") + mode = path.stat().st_mode + if mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH): + kinds.append("x") + except OSError: + return kinds + return kinds + + +def file_match(path: Path, base: Path, *, include_errors: bool) -> FileMatch: + """Convert a Path into the stable FileMatch shape.""" + absolute_path = _absolute(path) + absolute_base = _absolute(base) + errors: list[ErrorInfo] = [] + size = 0 + mtime = datetime.fromtimestamp(0, tz=UTC) + + try: + file_stat = path.stat() + size = file_stat.st_size if path.is_file() else 0 + mtime = datetime.fromtimestamp(file_stat.st_mtime, tz=UTC) + except OSError as exc: + if include_errors: + errors.append(_error_from_oserror(exc, absolute_path)) + + return FileMatch( + path=absolute_path, + relative_path=_relative_path(absolute_path, absolute_base), + size=size, + mtime=mtime, + kinds=_path_kinds(path), + errors=errors, + ) + + +def collect_file_search( + paths: Iterable[Path], + *, + base: Path, + limit: int | None, + max_bytes: int | None, + include_errors: bool, + timeout_seconds: float | None = None, + strict_base: bool = False, +) -> FileSearchResult: + """Collect path matches into a FileSearchResult with truncation metadata.""" + results: list[FileMatch] = [] + errors: list[ErrorInfo] = [] + total = 0 + bytes_read = 0 + truncated = False + truncated_reason: str | None = None + deadline = deadline_from_timeout(timeout_seconds) + + for path in paths: + try: + check_timeout(deadline) + except TimeoutError: + truncated = True + truncated_reason = "timeout" + if include_errors: + errors.append(timeout_error()) + break + total += 1 + if limit is not None and len(results) >= limit: + truncated = True + truncated_reason = "limit" + break + + if strict_base and not _is_contained(path, base): + if include_errors: + errors.append(_strict_base_error(path, _absolute(base))) + continue + + match = file_match(path, base, include_errors=include_errors) + record_size = len(json.dumps(to_json_dict(match), separators=(",", ":")).encode()) + if max_bytes is not None and bytes_read + record_size > max_bytes: + truncated = True + truncated_reason = "max_bytes" + break + + results.append(match) + errors.extend(match.errors) + bytes_read += record_size + + return FileSearchResult( + results=results, + truncated=truncated, + total_files_searched=total, + bytes_read=bytes_read, + errors=errors, + truncated_reason=truncated_reason, + ) diff --git a/src/rglob/agent/mcp.py b/src/rglob/agent/mcp.py new file mode 100644 index 0000000..868aae6 --- /dev/null +++ b/src/rglob/agent/mcp.py @@ -0,0 +1,332 @@ +"""Stdio MCP server for the rglob agent API.""" + +import asyncio +from collections.abc import Awaitable, Callable +from pathlib import Path +from typing import Protocol, cast + +from rglob.agent import ( + CountOptions, + GrepOptions, + WalkOptions, + count, + find_duplicates, + grep_all, + search_all, + to_json_dict, +) +from rglob.agent._introspection import describe_command, json_ready +from rglob.agent._models import JsonValue + +ToolHandler = Callable[..., Awaitable[JsonValue]] +ToolDecorator = Callable[[ToolHandler], ToolHandler] +IntLike = str | int | float + + +class McpServer(Protocol): + """Small protocol covering the MCP SDK surface we use.""" + + def tool(self) -> ToolDecorator: + """Return a decorator that registers an async tool.""" + + async def run_stdio_async(self) -> None: + """Run the stdio transport.""" + + +ServerFactory = Callable[[str], McpServer] + + +def _load_server_factory() -> ServerFactory: + """Import the official MCP FastMCP class lazily. + + `FastMCP` is the high-level server API the rest of this module is + written against (it provides the `@server.tool()` decorator and + `run_stdio_async()` coroutine). The low-level `mcp.server.Server` + class uses a different `@server.list_tools()` / `@server.call_tool()` + pattern and would not work here without a substantial rewrite. + """ + try: + from mcp.server.fastmcp import FastMCP + except ImportError as exc: + raise RuntimeError("rglob mcp requires the optional mcp extra") from exc + return cast("ServerFactory", FastMCP) + + +def _walk_options(patterns: list[str], base: str, filters: dict[str, object]) -> WalkOptions: + """Build conservative WalkOptions from MCP tool input.""" + return WalkOptions( + patterns=patterns, + base=Path(base), + exclude=cast("list[str]", filters.get("exclude", [])), + max_depth=cast("int | None", filters.get("max_depth")), + hidden=bool(filters.get("hidden", False)), + limit=cast("int | None", filters.get("limit", 5000)), + max_bytes=cast("int | None", filters.get("max_bytes")), + max_file_size=cast("int | None", filters.get("max_file_size")), + strict_base=True, + follow_symlinks=bool(filters.get("follow_symlinks", False)), + respect_gitignore=bool(filters.get("respect_gitignore", False)), + include_errors=bool(filters.get("include_errors", True)), + case_sensitive=cast("bool | None", filters.get("case_sensitive")), + sort=bool(filters.get("sort", True)), + ) + + +def _grep_options( + pattern: str, + paths: list[str], + base: str, + filters: dict[str, object], +) -> GrepOptions: + """Build conservative GrepOptions from MCP tool input.""" + walk = _walk_options(paths, base, filters) + return GrepOptions( + pattern=pattern, + paths=walk.patterns, + base=walk.base, + exclude=walk.exclude, + max_depth=walk.max_depth, + hidden=walk.hidden, + limit=walk.limit, + max_bytes=walk.max_bytes, + max_file_size=walk.max_file_size, + strict_base=walk.strict_base, + follow_symlinks=walk.follow_symlinks, + respect_gitignore=walk.respect_gitignore, + include_errors=walk.include_errors, + case_sensitive=walk.case_sensitive, + sort=walk.sort, + fixed_string=bool(filters.get("fixed_string", False)), + ignore_case=bool(filters.get("ignore_case", False)), + context=int(cast("IntLike", filters.get("context", 0))), + before=int(cast("IntLike", filters.get("before", 0))), + after=int(cast("IntLike", filters.get("after", 0))), + max_count=cast("int | None", filters.get("max_count")), + word=bool(filters.get("word", False)), + invert=bool(filters.get("invert", False)), + encoding=str(filters.get("encoding", "utf-8")), + text=bool(filters.get("text", False)), + ) + + +def _count_options(pattern: str, base: str, filters: dict[str, object]) -> CountOptions: + """Build conservative CountOptions from MCP tool input.""" + walk = _walk_options([pattern], base, filters) + return CountOptions( + patterns=walk.patterns, + base=walk.base, + exclude=walk.exclude, + max_depth=walk.max_depth, + hidden=walk.hidden, + limit=walk.limit, + max_bytes=walk.max_bytes, + max_file_size=walk.max_file_size, + strict_base=walk.strict_base, + follow_symlinks=walk.follow_symlinks, + respect_gitignore=walk.respect_gitignore, + include_errors=walk.include_errors, + case_sensitive=walk.case_sensitive, + sort=walk.sort, + no_empty=bool(filters.get("no_empty", False)), + no_comments=bool(filters.get("no_comments", False)), + encoding=str(filters.get("encoding", "utf-8")), + ) + + +def create_server(server_factory: ServerFactory | None = None) -> McpServer: + """Create and register the rglob MCP server. + + Tool signatures intentionally inline every filter as a named keyword + argument rather than using `**filters`. FastMCP introspects the + function signature to build the public JSON Schema for the tool; + `**kwargs` would render as a single opaque required `filters` dict + that agents cannot discover the shape of, and would also break call + validation (FastMCP requires every parameter to be explicit). + """ + factory = server_factory or _load_server_factory() + server = factory("rglob") + + @server.tool() + async def find_files( + pattern: str, + base: str = ".", + exclude: list[str] | None = None, + max_depth: int | None = None, + hidden: bool = False, + follow_symlinks: bool = False, + case_sensitive: bool | None = None, + respect_gitignore: bool = False, + limit: int | None = 5000, + max_bytes: int | None = None, + max_file_size: int | None = None, + ) -> JsonValue: + opts = _walk_options( + [pattern], + base, + _walk_filter_dict( + exclude=exclude, + max_depth=max_depth, + hidden=hidden, + follow_symlinks=follow_symlinks, + case_sensitive=case_sensitive, + respect_gitignore=respect_gitignore, + limit=limit, + max_bytes=max_bytes, + max_file_size=max_file_size, + ), + ) + return to_json_dict(search_all(opts)) + + @server.tool() + async def grep_content( + pattern: str, + paths: list[str] | None = None, + base: str = ".", + fixed_string: bool = False, + ignore_case: bool = False, + context: int = 0, + before: int = 0, + after: int = 0, + max_count: int | None = None, + word: bool = False, + invert: bool = False, + encoding: str = "utf-8", + text: bool = False, + exclude: list[str] | None = None, + max_depth: int | None = None, + hidden: bool = False, + follow_symlinks: bool = False, + case_sensitive: bool | None = None, + respect_gitignore: bool = False, + limit: int | None = 5000, + max_bytes: int | None = None, + max_file_size: int | None = None, + ) -> JsonValue: + filters = _walk_filter_dict( + exclude=exclude, + max_depth=max_depth, + hidden=hidden, + follow_symlinks=follow_symlinks, + case_sensitive=case_sensitive, + respect_gitignore=respect_gitignore, + limit=limit, + max_bytes=max_bytes, + max_file_size=max_file_size, + ) + filters.update( + { + "fixed_string": fixed_string, + "ignore_case": ignore_case, + "context": context, + "before": before, + "after": after, + "max_count": max_count, + "word": word, + "invert": invert, + "encoding": encoding, + "text": text, + } + ) + opts = _grep_options(pattern, paths or ["*"], base, filters) + return to_json_dict(grep_all(opts)) + + @server.tool() + async def count_lines( + pattern: str, + base: str = ".", + no_empty: bool = False, + no_comments: bool = False, + encoding: str = "utf-8", + exclude: list[str] | None = None, + max_depth: int | None = None, + hidden: bool = False, + follow_symlinks: bool = False, + case_sensitive: bool | None = None, + respect_gitignore: bool = False, + limit: int | None = 5000, + max_bytes: int | None = None, + max_file_size: int | None = None, + ) -> JsonValue: + filters = _walk_filter_dict( + exclude=exclude, + max_depth=max_depth, + hidden=hidden, + follow_symlinks=follow_symlinks, + case_sensitive=case_sensitive, + respect_gitignore=respect_gitignore, + limit=limit, + max_bytes=max_bytes, + max_file_size=max_file_size, + ) + filters.update({"no_empty": no_empty, "no_comments": no_comments, "encoding": encoding}) + opts = _count_options(pattern, base, filters) + return to_json_dict(count(opts)) + + @server.tool() + async def find_duplicate_files( + pattern: str, + base: str = ".", + exclude: list[str] | None = None, + max_depth: int | None = None, + hidden: bool = False, + follow_symlinks: bool = False, + case_sensitive: bool | None = None, + respect_gitignore: bool = False, + limit: int | None = 5000, + max_bytes: int | None = None, + max_file_size: int | None = None, + ) -> JsonValue: + opts = _walk_options( + [pattern], + base, + _walk_filter_dict( + exclude=exclude, + max_depth=max_depth, + hidden=hidden, + follow_symlinks=follow_symlinks, + case_sensitive=case_sensitive, + respect_gitignore=respect_gitignore, + limit=limit, + max_bytes=max_bytes, + max_file_size=max_file_size, + ), + ) + return to_json_dict(find_duplicates(opts)) + + @server.tool() + async def describe_subcommand(name: str) -> JsonValue: + return json_ready(describe_command(name)) + + return server + + +def _walk_filter_dict( + *, + exclude: list[str] | None, + max_depth: int | None, + hidden: bool, + follow_symlinks: bool, + case_sensitive: bool | None, + respect_gitignore: bool, + limit: int | None, + max_bytes: int | None, + max_file_size: int | None, +) -> dict[str, object]: + """Collect the shared walker filter kwargs into a `_walk_options`-friendly dict.""" + return { + "exclude": exclude or [], + "max_depth": max_depth, + "hidden": hidden, + "follow_symlinks": follow_symlinks, + "case_sensitive": case_sensitive, + "respect_gitignore": respect_gitignore, + "limit": limit, + "max_bytes": max_bytes, + "max_file_size": max_file_size, + } + + +def main() -> None: + """Run the stdio MCP server.""" + server = create_server() + asyncio.run(server.run_stdio_async()) diff --git a/src/rglob/cli.py b/src/rglob/cli.py new file mode 100644 index 0000000..3a63677 --- /dev/null +++ b/src/rglob/cli.py @@ -0,0 +1,896 @@ +"""Typer-powered command-line interface for `rglob`. + +This is the Phase 4 rewrite. The subcommand surface (`find`, `lcount`, +`tsize`) and their legacy flags (`--base`, `--no-empty`, `--no-comments`, +`--unit`) are byte-compatible with the 1.x argparse CLI. New filter and +output flags are layered on top per the modernization roadmap. +""" + +from __future__ import annotations + +import json +import sys +from collections.abc import Callable, Iterable +from pathlib import Path +from typing import Annotated, NoReturn + +import typer +from rich.console import Console + +from rglob import find as _find +from rglob._count import count_all +from rglob._grep import grep_all +from rglob.agent._introspection import ( + all_schemas, + capability_report, + command_schema, + describe_command, + json_ready, + unknown_command_envelope, +) +from rglob.agent._models import ( + AGENT_API_VERSION, + CountOptions, + ErrorCode, + ErrorInfo, + GrepOptions, + to_json_dict, +) +from rglob.agent._runtime import collect_file_search +from rglob.rglob import ( + gigabytes, + kilobytes, + megabytes, + terabytes, +) +from rglob.rglob import ( + lcount as _lcount, +) +from rglob.rglob import ( + tsize as _tsize, +) + +_console = Console() +_err = Console(stderr=True) + +app = typer.Typer( + no_args_is_help=True, + rich_markup_mode="rich", + pretty_exceptions_enable=False, + help="Recursive file operations: find, lcount, tsize.", + epilog=( + "Quote your patterns so the shell doesn't pre-expand them.\n" + 'Example: rglob find "*.py" (not rglob find *.py).' + ), +) + + +# ─── Shared filter options ──────────────────────────────────────────────────── + +# Repeated as Annotated aliases so each command can opt in selectively without +# stringly-typed `**kwargs` plumbing. +BaseOpt = Annotated[Path, typer.Option("--base", help="Base directory to walk.")] +ExcludeOpt = Annotated[ + list[str], + typer.Option("-E", "--exclude", help="Glob(s) to exclude (and prune from descent)."), +] +MaxDepthOpt = Annotated[ + int | None, + typer.Option("-d", "--max-depth", min=0, help="Maximum recursion depth (None=unbounded)."), +] +HiddenOpt = Annotated[ + bool, + typer.Option("-H", "--hidden", help="Include dotfiles and dot-directories."), +] +FollowOpt = Annotated[ + bool, + typer.Option("-L", "--follow", help="Follow symlinks (cycles are auto-detected)."), +] +CaseSensitiveOpt = Annotated[ + bool | None, + typer.Option( + "-s/-i", + "--case-sensitive/--case-insensitive", + help="Force case sensitivity (default: follow host OS).", + ), +] +GitignoreOpt = Annotated[ + bool, + typer.Option( + "--gitignore/--no-gitignore", + help="Honour .gitignore files (lands fully in Phase 5).", + ), +] + + +def _detect_pre_expansion(patterns: Iterable[str]) -> None: + """Warn if the shell appears to have pre-expanded glob patterns.""" + pats = list(patterns) + if len(pats) > 1 and not any(any(c in p for c in "*?[") for p in pats): + _err.print( + "[yellow]warning:[/yellow] received multiple patterns with no " + "glob metacharacters — did your shell pre-expand them? Quote " + 'with "..." to avoid this.', + highlight=False, + ) + + +def _resolve_case_sensitive(value: bool | None) -> bool | None: + """Pass-through; kept as a seam in case future flags want to override.""" + return value + + +def _emit_json(payload: object) -> None: + """Write stable JSON to stdout without Rich styling.""" + sys.stdout.write(json.dumps(json_ready(payload), indent=2, sort_keys=True) + "\n") + + +def _emit_jsonl(payload: object) -> None: + """Write one compact JSON object line.""" + sys.stdout.write(json.dumps(json_ready(payload), sort_keys=True) + "\n") + + +def _emit_grep_jsonl_stream(opts: GrepOptions) -> None: + """Stream grep matches as JSONL records, then a final summary line. + + Each match emits as a compact JSON object on its own line (so agents + can begin processing without buffering the whole result). When the + search completes, a final summary line is emitted with `truncated`, + `truncated_reason`, `total_files_searched`, `bytes_read`, and + `errors` so the consumer knows the stream is complete and how much + work happened. Each record carries a `kind` field (``"match"`` or + ``"summary"``) for easy dispatch. + """ + # Single pass over the underlying search; we iterate the materialised + # results list so future refactors that make `grep_iter` truly lazy + # can swap in here without changing the wire format. + result = grep_all(opts) + for match in result.results: + # `json_ready` returns a JsonValue union; for a dataclass input + # it's always a dict. The cast keeps mypy strict happy without + # narrowing the return-type contract of `json_ready` itself. + from typing import cast + + match_payload = cast("dict[str, object]", json_ready(match)) + sys.stdout.write(json.dumps({"kind": "match", **match_payload}, sort_keys=True) + "\n") + sys.stdout.flush() + sys.stdout.write( + json.dumps( + { + "kind": "summary", + "truncated": result.truncated, + "truncated_reason": result.truncated_reason, + "total_files_searched": result.total_files_searched, + "bytes_read": result.bytes_read, + "errors": [json_ready(e) for e in result.errors], + }, + sort_keys=True, + ) + + "\n" + ) + + +def _exit_bad_predicate(message: str, *, structured: bool) -> NoReturn: + """Exit with either a machine-readable or human-readable predicate error.""" + if structured: + _emit_json( + { + "ok": False, + "error": to_json_dict(ErrorInfo(ErrorCode.BAD_PREDICATE, message, None)), + } + ) + raise typer.Exit(code=2) + _err.print(f"[red]error:[/red] {message}") + raise typer.Exit(code=2) + + +def _parse_size_option(value: str | None, *, structured: bool) -> int | None: + """Parse a size option for structured commands.""" + if value is None: + return None + from rglob._filters import parse_size + + try: + return parse_size(value) + except ValueError as exc: + _exit_bad_predicate(str(exc), structured=structured) + + +def _format_path(path: Path, fmt: str | None, base: Path) -> str: + """Render a path according to the optional `--format` template.""" + if fmt is None: + return str(path) + try: + size = path.stat().st_size if path.is_file() else 0 + except OSError: + size = 0 + return fmt.format( + path=path, + rel=path.relative_to(base) if path.is_absolute() else path, + name=path.name, + size=size, + size_kb=kilobytes(size), + size_mb=megabytes(size), + ) + + +# ─── find ───────────────────────────────────────────────────────────────────── + + +@app.command(help="List paths matching one or more glob patterns.") +def find( + patterns: Annotated[ + list[str], + typer.Argument( + help='Glob pattern(s). Quote them, e.g. "*.py".', + show_default=False, + ), + ], + base: BaseOpt = Path.cwd(), + exclude: ExcludeOpt = [], + max_depth: MaxDepthOpt = None, + hidden: HiddenOpt = False, + follow: FollowOpt = False, + case_sensitive: CaseSensitiveOpt = None, + gitignore: GitignoreOpt = False, + kind: Annotated[ + list[str], + typer.Option( + "-t", + "--type", + help='Match only "f"=file, "d"=dir, "l"=symlink, "x"=executable. Repeatable.', + ), + ] = [], + min_size: Annotated[ + str | None, + typer.Option("--min-size", help='Minimum file size, e.g. "1K", "5MiB".'), + ] = None, + max_size: Annotated[ + str | None, + typer.Option("--max-size", help="Maximum file size."), + ] = None, + newer_than: Annotated[ + str | None, + typer.Option("--newer-than", help='ISO date or duration like "7d".'), + ] = None, + older_than: Annotated[ + str | None, + typer.Option("--older-than", help='ISO date or duration like "30d".'), + ] = None, + newer_than_file: Annotated[ + Path | None, + typer.Option("--newer-than-file", help="Only entries newer than this file."), + ] = None, + perm: Annotated[ + str | None, + typer.Option("--perm", help='POSIX mode filter, e.g. "644", "-111", "/222".'), + ] = None, + uid: Annotated[ + int | None, + typer.Option("--uid", help="POSIX owner uid filter."), + ] = None, + gid: Annotated[ + int | None, + typer.Option("--gid", help="POSIX owner gid filter."), + ] = None, + json_out: Annotated[ + bool, + typer.Option("--json", help="Emit a FileSearchResult JSON object."), + ] = False, + jsonl: Annotated[ + bool, + typer.Option( + "--jsonl", + help="Emit one compact FileSearchResult JSON object line.", + ), + ] = False, + limit: Annotated[ + int | None, + typer.Option("--limit", min=1, help="Maximum structured records to emit."), + ] = None, + max_bytes: Annotated[ + str | None, + typer.Option("--max-bytes", help="Maximum structured output bytes."), + ] = None, + max_file_size: Annotated[ + str | None, + typer.Option("--max-file-size", help="Skip files larger than this."), + ] = None, + include_errors: Annotated[ + bool, + typer.Option( + "--include-errors/--no-include-errors", + help="Include per-record errors in structured output.", + ), + ] = True, + null: Annotated[ + bool, + typer.Option("-0", "--null", help="Separate paths with NUL bytes (xargs -0)."), + ] = False, + fmt: Annotated[ + str | None, + typer.Option( + "--format", + help='Mini-template, e.g. "{path} {size_mb:.2f} MiB".', + ), + ] = None, +) -> None: + """Print paths under ``base`` matching ``patterns``.""" + _detect_pre_expansion(patterns) + from typing import cast + + from rglob._filters import Kind + + max_bytes_value = _parse_size_option(max_bytes, structured=json_out or jsonl) + + matches = _find( + base, + patterns, + exclude=tuple(exclude), + max_depth=max_depth, + hidden=hidden, + follow_symlinks=follow, + case_sensitive=_resolve_case_sensitive(case_sensitive), + on_error="ignore" if json_out or jsonl else "warn", + kinds=cast("list[Kind]", kind), + min_size=min_size, + max_size=max_file_size or max_size, + newer_than=newer_than, + older_than=older_than, + newer_than_file=newer_than_file, + perm=perm, + uid=uid, + gid=gid, + respect_gitignore=gitignore, + ) + + if json_out: + _emit_json( + collect_file_search( + matches, + base=base, + limit=limit, + max_bytes=max_bytes_value, + include_errors=include_errors, + ) + ) + return + + if jsonl: + _emit_jsonl( + collect_file_search( + matches, + base=base, + limit=limit, + max_bytes=max_bytes_value, + include_errors=include_errors, + ) + ) + return + + if null: + for path in matches: + sys.stdout.write(str(path) + "\0") + return + + for path in matches: + print(_format_path(path, fmt, base)) + + +# ─── grep / count ──────────────────────────────────────────────────────────── + + +@app.command(help="Search file contents for a regex or fixed string.") +def grep( + pattern: Annotated[str, typer.Argument(help="Regex or fixed string to search for.")], + files_or_globs: Annotated[ + list[str], + typer.Argument(help='File glob(s) to search, e.g. "src/**/*.py".'), + ] = [], + base: BaseOpt = Path.cwd(), + exclude: ExcludeOpt = [], + max_depth: MaxDepthOpt = None, + hidden: HiddenOpt = False, + follow: FollowOpt = False, + case_sensitive: CaseSensitiveOpt = None, + gitignore: GitignoreOpt = False, + fixed_string: Annotated[ + bool, + typer.Option("-F", "--fixed-string", help="Treat pattern as a literal string."), + ] = False, + ignore_case: Annotated[ + bool, + typer.Option("-i", "--ignore-case", help="Match content case-insensitively."), + ] = False, + context: Annotated[int, typer.Option("-C", "--context", min=0, help="Context lines.")] = 0, + before: Annotated[int, typer.Option("-B", "--before", min=0, help="Lines before.")] = 0, + after: Annotated[int, typer.Option("-A", "--after", min=0, help="Lines after.")] = 0, + max_count: Annotated[ + int | None, + typer.Option("-m", "--max-count", min=1, help="Maximum matches to return."), + ] = None, + word: Annotated[bool, typer.Option("-w", "--word", help="Match whole words.")] = False, + invert: Annotated[bool, typer.Option("-v", "--invert", help="Invert the match.")] = False, + encoding: Annotated[str, typer.Option("--encoding", help="Text encoding.")] = "utf-8", + text: Annotated[ + bool, + typer.Option("-a", "--text", help="Search binary files as text."), + ] = False, + limit: Annotated[ + int | None, + typer.Option("--limit", min=1, help="Maximum structured matches to emit."), + ] = None, + max_bytes: Annotated[ + str | None, + typer.Option("--max-bytes", help="Maximum bytes to read."), + ] = None, + max_file_size: Annotated[ + str | None, + typer.Option("--max-file-size", help="Skip files larger than this."), + ] = None, + files_with_matches_flag: Annotated[ + bool, + typer.Option( + "-l", + "--files-with-matches", + help="Emit one record per matching file, no per-line content.", + ), + ] = False, + count_only_flag: Annotated[ + bool, + typer.Option( + "-c", + "--count-only", + help="Emit one record per matching file with the match count.", + ), + ] = False, + json_out: Annotated[bool, typer.Option("--json", help="Emit a LineSearchResult.")] = False, + jsonl: Annotated[ + bool, + typer.Option( + "--jsonl", + help="Stream one LineMatch per line, then a final summary line.", + ), + ] = False, +) -> None: + """Search matching files and print content matches.""" + if files_with_matches_flag and count_only_flag: + _err.print( + "[red]error:[/red] --files-with-matches and --count-only are mutually exclusive", + highlight=False, + ) + raise typer.Exit(code=2) + structured = json_out or jsonl + opts = GrepOptions( + pattern=pattern, + paths=files_or_globs or ["*"], + base=base, + exclude=exclude, + max_depth=max_depth, + hidden=hidden, + follow_symlinks=follow, + case_sensitive=_resolve_case_sensitive(case_sensitive), + respect_gitignore=gitignore, + fixed_string=fixed_string, + ignore_case=ignore_case, + context=context, + before=before, + after=after, + max_count=max_count, + word=word, + invert=invert, + encoding=encoding, + text=text, + limit=limit, + max_bytes=_parse_size_option(max_bytes, structured=structured), + max_file_size=_parse_size_option(max_file_size, structured=structured), + files_with_matches=files_with_matches_flag, + count_only=count_only_flag, + ) + + # Streaming JSONL emits each match as it's found, then a final + # summary record so consumers don't have to buffer the whole result. + if jsonl: + _emit_grep_jsonl_stream(opts) + return + + result = grep_all(opts) + + if json_out: + _emit_json(result) + return + + if files_with_matches_flag: + for match in result.results: + print(match.path) + elif count_only_flag: + for match in result.results: + print(f"{match.path}:{match.line_number}") + else: + for match in result.results: + print(f"{match.path}:{match.line_number}:{match.content}") + for error in result.errors: + _err.print(f"[yellow]warning:[/yellow] {error.code}: {error.message}", highlight=False) + + +@app.command(help="Count files, lines, and bytes for matching files.") +def count( + patterns: Annotated[ + list[str], + typer.Argument(help='Glob pattern(s), e.g. "*.py".', show_default=False), + ], + base: BaseOpt = Path.cwd(), + exclude: ExcludeOpt = [], + max_depth: MaxDepthOpt = None, + hidden: HiddenOpt = False, + follow: FollowOpt = False, + case_sensitive: CaseSensitiveOpt = None, + gitignore: GitignoreOpt = False, + no_empty: Annotated[bool, typer.Option("--no-empty", help="Skip empty lines.")] = False, + no_comments: Annotated[ + bool, + typer.Option("--no-comments", help="Skip lines starting with `#`."), + ] = False, + encoding: Annotated[str, typer.Option("--encoding", help="Text encoding.")] = "utf-8", + limit: Annotated[ + int | None, + typer.Option("--limit", min=1, help="Maximum files to count."), + ] = None, + max_bytes: Annotated[ + str | None, + typer.Option("--max-bytes", help="Maximum bytes to read."), + ] = None, + max_file_size: Annotated[ + str | None, + typer.Option("--max-file-size", help="Skip files larger than this."), + ] = None, + json_out: Annotated[bool, typer.Option("--json", help="Emit a Stats object.")] = False, + jsonl: Annotated[ + bool, + typer.Option("--jsonl", help="Emit one compact Stats object line."), + ] = False, +) -> None: + """Print structured file, line, and byte counts.""" + structured = json_out or jsonl + opts = CountOptions( + patterns=patterns, + base=base, + exclude=exclude, + max_depth=max_depth, + hidden=hidden, + follow_symlinks=follow, + case_sensitive=_resolve_case_sensitive(case_sensitive), + respect_gitignore=gitignore, + no_empty=no_empty, + no_comments=no_comments, + encoding=encoding, + limit=limit, + max_bytes=_parse_size_option(max_bytes, structured=structured), + max_file_size=_parse_size_option(max_file_size, structured=structured), + ) + result = count_all(opts) + + if json_out: + _emit_json(result) + return + if jsonl: + _emit_jsonl(result) + return + + print(f"Files: {result.files}") + print(f"Lines: {result.lines}") + print(f"Bytes: {result.bytes}") + + +# ─── lcount ─────────────────────────────────────────────────────────────────── + + +def _line_filter(no_empty: bool, no_comments: bool) -> Callable[[str], bool]: + """Build the line predicate used by `lcount`.""" + predicates: list[Callable[[str], bool]] = [] + if no_empty: + predicates.append(lambda line: bool(line.strip())) + if no_comments: + predicates.append(lambda line: not line.strip().startswith("#")) + if not predicates: + return lambda _line: True + return lambda line: all(p(line) for p in predicates) + + +@app.command(help="Count lines across files matching a glob pattern.") +def lcount( + pattern: Annotated[str, typer.Argument(help='Glob pattern, e.g. "*.py".')], + base: BaseOpt = Path.cwd(), + no_empty: Annotated[bool, typer.Option("--no-empty", help="Skip empty lines.")] = False, + no_comments: Annotated[ + bool, + typer.Option("--no-comments", help="Skip lines starting with `#`."), + ] = False, +) -> None: + """Print the total line count.""" + count = _lcount(str(base), pattern, func=_line_filter(no_empty, no_comments)) + print(f"Total lines: {count}") + + +# ─── tsize ──────────────────────────────────────────────────────────────────── + +_UNIT_FUNCS: dict[str, Callable[[float], float]] = { + "kb": kilobytes, + "mb": megabytes, + "gb": gigabytes, + "tb": terabytes, +} + + +@app.command(help="Sum the total size of files matching a glob pattern.") +def tsize( + pattern: Annotated[str, typer.Argument(help='Glob pattern, e.g. "*.py".')], + base: BaseOpt = Path.cwd(), + unit: Annotated[ + str, + typer.Option("--unit", help="Unit (kb/mb/gb/tb).", case_sensitive=False), + ] = "mb", +) -> None: + """Print the total size in the chosen unit.""" + unit_key = unit.lower() + if unit_key not in _UNIT_FUNCS: + _err.print( + f"[red]error:[/red] unknown unit '{unit}' (expected one of: {', '.join(_UNIT_FUNCS)})" + ) + raise typer.Exit(code=2) + total = _tsize(str(base), pattern, func=_UNIT_FUNCS[unit_key]) + print(f"Total size: {total:.2f} {unit_key.upper()}") + + +# ─── stats / tree / top / dupes (Phase 5 fun features) ──────────────────────── + + +@app.command(help="Summarise a glob result: count, total size, extension breakdown.") +def stats( + pattern: Annotated[str, typer.Argument(help='Glob pattern, e.g. "*.py".')] = "*", + base: BaseOpt = Path.cwd(), + hidden: HiddenOpt = False, + follow: FollowOpt = False, +) -> None: + """Print a Rich summary table for the matching files.""" + from collections import Counter + + from rich.table import Table + + paths = list( + _find( + base, + pattern, + hidden=hidden, + follow_symlinks=follow, + kinds=("f",), + on_error="warn", + ) + ) + total_bytes = 0 + ext_counter: Counter[str] = Counter() + ext_bytes: Counter[str] = Counter() + for p in paths: + try: + size = p.stat().st_size + except OSError: + size = 0 + ext = p.suffix.lower() or "(none)" + total_bytes += size + ext_counter[ext] += 1 + ext_bytes[ext] += size + + summary = Table(title=f"stats: {pattern!r} under {base}") + summary.add_column("Metric", style="cyan") + summary.add_column("Value", justify="right") + summary.add_row("Files", str(len(paths))) + summary.add_row("Total size", f"{megabytes(total_bytes):.2f} MiB") + summary.add_row("Largest extension", ext_counter.most_common(1)[0][0] if ext_counter else "—") + _console.print(summary) + + if ext_counter: + breakdown = Table(title="By extension") + breakdown.add_column("Ext", style="green") + breakdown.add_column("Count", justify="right") + breakdown.add_column("Size (MiB)", justify="right") + for ext, count in ext_counter.most_common(): + breakdown.add_row(ext, str(count), f"{megabytes(ext_bytes[ext]):.2f}") + _console.print(breakdown) + + +@app.command(help="Render a Unicode tree of matches (depth-limited by default).") +def tree( + pattern: Annotated[str, typer.Argument(help='Glob pattern, e.g. "*".')] = "*", + base: BaseOpt = Path.cwd(), + max_depth: MaxDepthOpt = 3, + hidden: HiddenOpt = False, +) -> None: + """Print a Rich tree of matching paths under ``base``.""" + from rich.tree import Tree + + paths = sorted( + _find( + base, + pattern, + max_depth=max_depth, + hidden=hidden, + on_error="warn", + ) + ) + + root = Tree(f"[bold]{base}[/bold]") + nodes: dict[Path, Tree] = {base: root} + for path in paths: + parents: list[Path] = [] + cur = path.parent + while cur != base and cur.parent != cur: + parents.append(cur) + cur = cur.parent + parents.reverse() + for parent in parents: + if parent not in nodes: + nodes[parent] = nodes[parent.parent].add(f"[blue]{parent.name}/[/blue]") + marker = "📁" if path.is_dir() else "📄" + nodes[path] = nodes[path.parent].add(f"{marker} {path.name}") + _console.print(root) + + +@app.command(help="Show the top-N largest files matching a pattern.") +def top( + pattern: Annotated[str, typer.Argument(help='Glob pattern, e.g. "*.log".')] = "*", + base: BaseOpt = Path.cwd(), + n: Annotated[int, typer.Option("-n", "--n", min=1, help="Number of entries.")] = 10, + hidden: HiddenOpt = False, +) -> None: + """Print a Rich table of the N largest files.""" + from rich.table import Table + + sized: list[tuple[int, Path]] = [] + for p in _find(base, pattern, hidden=hidden, kinds=("f",), on_error="warn"): + try: + sized.append((p.stat().st_size, p)) + except OSError: + continue + sized.sort(reverse=True) + top_n = sized[:n] + + table = Table(title=f"Top {n} largest files matching {pattern!r}") + table.add_column("Rank", justify="right", style="cyan") + table.add_column("Size (MiB)", justify="right") + table.add_column("Path", style="green") + for rank, (size, path) in enumerate(top_n, start=1): + table.add_row(str(rank), f"{megabytes(size):.3f}", path.name) + _console.print(table) + + +@app.command(help="Find groups of duplicate files (by content hash).") +def dupes( + pattern: Annotated[str, typer.Argument(help='Glob pattern, e.g. "*".')] = "*", + base: BaseOpt = Path.cwd(), + min_size: Annotated[ + str | None, + typer.Option( + "--min-size", + help='Ignore files smaller than this (e.g. "1K").', + ), + ] = None, + hidden: HiddenOpt = False, +) -> None: + """Print groups of duplicate files under ``base``.""" + from rich.table import Table + + from rglob._dupes import find_duplicates + + files = list( + _find( + base, + pattern, + kinds=("f",), + hidden=hidden, + min_size=min_size, + on_error="warn", + ) + ) + groups = find_duplicates(files) + if not groups: + _console.print("[green]No duplicates found.[/green]") + return + + table = Table(title=f"Duplicate groups: {len(groups)}") + table.add_column("Group", justify="right", style="cyan") + table.add_column("Size (MiB)", justify="right") + table.add_column("Paths", style="green") + for idx, group in enumerate(groups, start=1): + try: + size = group[0].stat().st_size + except OSError: # pragma: no cover - races between scan and display + size = 0 + table.add_row(str(idx), f"{megabytes(size):.3f}", "\n".join(p.name for p in group)) + _console.print(table) + + +# ─── Agent introspection ───────────────────────────────────────────────────── + + +@app.command(help="Describe a subcommand as a stable JSON manifest.") +def describe( + subcommand: Annotated[str, typer.Argument(help="Subcommand to describe.")], +) -> None: + """Print a machine-readable command manifest.""" + try: + _emit_json(describe_command(subcommand)) + except ValueError: + _emit_json(unknown_command_envelope(subcommand)) + raise typer.Exit(code=2) from None + + +@app.command(help="Print input and output JSON Schemas for a subcommand.") +def schema( + subcommand: Annotated[str | None, typer.Argument(help="Subcommand to inspect.")] = None, + all_: Annotated[ + bool, + typer.Option("--all", help="Print every public agent JSON Schema."), + ] = False, +) -> None: + """Print a command's input and output schemas.""" + if all_: + _emit_json(all_schemas()) + return + if subcommand is None: + _emit_json(unknown_command_envelope("")) + raise typer.Exit(code=2) from None + try: + _emit_json(command_schema(subcommand)) + except ValueError: + _emit_json(unknown_command_envelope(subcommand)) + raise typer.Exit(code=2) from None + + +@app.command(help="Report installed agent-facing capabilities as JSON.") +def capabilities( + json_out: Annotated[ + bool, + typer.Option( + "--json", + help="Accepted for explicit machine callers; output is always JSON.", + ), + ] = False, +) -> None: + """Print the agent capability report.""" + _ = json_out + _emit_json(capability_report()) + + +@app.command(help="Print the current agent API version as JSON.") +def agent_version() -> None: + """Print the current agent API version.""" + _emit_json(AGENT_API_VERSION) + + +@app.command(help="Run the rglob stdio MCP server.") +def mcp() -> None: + """Launch the optional stdio MCP server.""" + try: + from rglob.agent.mcp import main as mcp_main + + mcp_main() + except RuntimeError as exc: + _emit_json( + { + "ok": False, + "error": to_json_dict(ErrorInfo(ErrorCode.UNSUPPORTED_PLATFORM, str(exc), None)), + } + ) + raise typer.Exit(code=2) from exc + + +# ─── Entry point ────────────────────────────────────────────────────────────── + + +def main() -> None: + """Entry point referenced by `[project.scripts] rglob` in pyproject.toml.""" + app() + + +if __name__ == "__main__": # pragma: no cover - guarded `python -m` entry + main() diff --git a/src/rglob/py.typed b/src/rglob/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/rglob/rglob.py b/src/rglob/rglob.py new file mode 100644 index 0000000..ea11a78 --- /dev/null +++ b/src/rglob/rglob.py @@ -0,0 +1,532 @@ +"""Recursive filesystem walking, line counting, and size aggregation. + +The package's primary public surface: + +- :func:`find` — lazy generator yielding :class:`pathlib.Path` matches +- :func:`find_all` — eager :class:`list` variant of :func:`find` +- :func:`rglob` / :func:`rglob_` — legacy wrappers returning ``list[str]`` +- :func:`lcount` — line counting across matching files +- :func:`tsize` — total size aggregation with unit conversion +- :func:`kilobytes` / :func:`megabytes` / :func:`gigabytes` / :func:`terabytes` + — binary-prefix conversion helpers +""" + +import fnmatch +import math +import os +import re +import sys +import warnings +from collections.abc import Callable, Iterable, Iterator, Sequence +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Literal + +from rglob._filters import ( + Kind, + gitignore_matcher, + kinds_predicate, + mtime_predicate, + owner_predicate, + perm_predicate, + size_predicate, +) + +__all__ = [ + "find", + "find_all", + "gigabytes", + "kilobytes", + "lcount", + "megabytes", + "rglob", + "rglob_", + "terabytes", + "tsize", +] + +# Type alias for the on_error knob. +OnError = Literal["ignore", "warn", "raise"] + + +# ─── Unit helpers (kept from the 1.x API, reused by Phase 5 size parser) ────── + + +def kilobytes(value: float) -> float: + """Convert bytes to kibibytes (KiB).""" + return value / math.pow(2.0, 10.0) + + +def megabytes(value: float) -> float: + """Convert bytes to mebibytes (MiB).""" + return value / math.pow(2.0, 20.0) + + +def gigabytes(value: float) -> float: + """Convert bytes to gibibytes (GiB).""" + return value / math.pow(2.0, 30.0) + + +def terabytes(value: float) -> float: + """Convert bytes to tebibytes (TiB).""" + return value / math.pow(2.0, 40.0) + + +# ─── Pattern compilation ────────────────────────────────────────────────────── + + +def _os_default_case_sensitive() -> bool: + """Return the platform-default case sensitivity for filename matching.""" + return sys.platform not in ("win32", "darwin") + + +def _always_true(_value: str) -> bool: + """Constant `True` matcher used as a fast path for `*` / `**` patterns.""" + return True + + +def _compile_matcher(pattern: str, *, case_sensitive: bool) -> tuple[Callable[[str], bool], bool]: + """Compile a single glob pattern into a matcher. + + Returns ``(matcher, is_path_pattern)``. ``is_path_pattern`` is ``True`` + when the pattern should match against the entry's *relative* path + (because it contains a separator or ``**``); otherwise the matcher runs + against the entry's basename. + + ``**`` is supported as "any number of path components, including zero". + All other meta-characters follow ``fnmatch`` semantics. + + Two patterns shortcut to a no-op matcher to skip regex compilation + and per-entry `fullmatch` calls: bare ``*`` (match every basename) + and bare ``**`` (match every relative path). + """ + if pattern == "*": + return (_always_true, False) + if pattern == "**": + return (_always_true, True) + + flags = 0 if case_sensitive else re.IGNORECASE + has_double_star = "**" in pattern + has_separator = "/" in pattern or os.sep in pattern + + if has_double_star: + # Translate `**/` (slash-suffix) and `/**` (slash-prefix) into + # placeholders that survive `fnmatch.translate` and then expand + # to "zero or more path components". Bare `**` (the standalone + # pattern) is handled by the shortcut at the top of the function; + # `**` between non-slash chars (`a**b`) is uncommon and falls + # through to the per-character path, where it ends up equivalent + # to a single `*`. + s_dstar_slash = "\x00DSTARSLASH\x00" + s_slash_dstar = "\x00SLASHDSTAR\x00" + cooked: list[str] = [] + i = 0 + normalized = pattern.replace(os.sep, "/") + while i < len(normalized): + if normalized[i : i + 3] == "**/": + cooked.append(s_dstar_slash) + i += 3 + elif normalized[i : i + 3] == "/**" and ( + i + 3 == len(normalized) or normalized[i + 3] == "/" + ): + cooked.append(s_slash_dstar) + i += 3 + else: + # Bare `**` between segments (e.g. `a**b`) falls through to + # the per-character path; fnmatch.translate turns `*` into + # `.*`, so `**` becomes `.*.*` — semantically equivalent + # to `.*`. Treating it specially here would be dead code: + # gitignore/ripgrep semantics require `**` to be its own + # path component, which is already handled by the `**/` + # and `/**` branches above. + cooked.append(normalized[i]) + i += 1 + translated = fnmatch.translate("".join(cooked)) + translated = translated.replace(s_dstar_slash, "(?:.*/)?") + translated = translated.replace(s_slash_dstar, "(?:/.*)?") + regex = re.compile(translated, flags) + return (lambda s: regex.fullmatch(s) is not None, True) + + if has_separator: + regex = re.compile(fnmatch.translate(pattern), flags) + return (lambda s: regex.fullmatch(s) is not None, True) + + # Basename pattern — common case (legacy `rglob(base, "*.py")`). + regex = re.compile(fnmatch.translate(pattern), flags) + return (lambda s: regex.fullmatch(s) is not None, False) + + +def _compile_patterns( + patterns: Sequence[str], *, case_sensitive: bool +) -> list[tuple[Callable[[str], bool], bool]]: + """Compile a sequence of glob patterns.""" + return [_compile_matcher(p, case_sensitive=case_sensitive) for p in patterns] + + +def _matches_any( + matchers: Iterable[tuple[Callable[[str], bool], bool]], + *, + basename: str, + rel_str: str, +) -> bool: + """Return ``True`` if any compiled matcher matches the entry.""" + for fn, is_path in matchers: + candidate = rel_str if is_path else basename + if fn(candidate): + return True + return False + + +# ─── Error handling ─────────────────────────────────────────────────────────── + + +def _make_error_handler(mode: OnError) -> Callable[[OSError], None]: + """Return a callable that disposes of OS errors per the chosen mode.""" + if mode == "raise": + + def _raise(exc: OSError) -> None: + raise exc + + return _raise + if mode == "warn": + + def _warn(exc: OSError) -> None: + warnings.warn(f"rglob walk: {exc}", RuntimeWarning, stacklevel=2) + + return _warn + # "ignore" + return lambda _exc: None + + +# ─── Core walker ────────────────────────────────────────────────────────────── + + +def _scandir_sorted( + path: Path, + *, + sort: bool, + on_error: Callable[[OSError], None], +) -> list[os.DirEntry[str]]: + """Return the entries of ``path`` (optionally sorted by name).""" + try: + with os.scandir(path) as it: + entries = list(it) + except OSError as exc: + on_error(exc) + return [] + if sort: + entries.sort(key=lambda e: e.name) + return entries + + +def _walk( + current: Path, + current_rel: str, + depth: int, + *, + matchers: list[tuple[Callable[[str], bool], bool]], + excluders: list[tuple[Callable[[str], bool], bool]], + max_depth: int | None, + hidden: bool, + follow_symlinks: bool, + sort: bool, + on_error: Callable[[OSError], None], + visited: set[str], + entry_filters: tuple[Callable[[os.DirEntry[str]], bool], ...], + gitignore: Callable[[Path], bool] | None, +) -> Iterator[Path]: + """Depth-first walker producing matching paths.""" + for entry in _scandir_sorted(current, sort=sort, on_error=on_error): + name = entry.name + if not hidden and name.startswith("."): + continue + + rel_str = f"{current_rel}/{name}" if current_rel else name + path: Path | None = None + + if excluders and _matches_any(excluders, basename=name, rel_str=rel_str): + continue + if gitignore is not None: + path = Path(entry.path) + if gitignore(path): + continue + + if (not matchers or _matches_any(matchers, basename=name, rel_str=rel_str)) and all( + check(entry) for check in entry_filters + ): + if path is None: + path = Path(entry.path) + yield path + + try: + is_dir = entry.is_dir(follow_symlinks=follow_symlinks) + except OSError as exc: + on_error(exc) + continue + if not is_dir: + continue + if max_depth is not None and depth >= max_depth: + continue + + if path is None: + path = Path(entry.path) + + if follow_symlinks: + try: + real = os.path.realpath(path) + except OSError as exc: # pragma: no cover - realpath is documented not to raise + on_error(exc) + continue + if real in visited: + continue + visited.add(real) + + yield from _walk( + path, + rel_str, + depth + 1, + matchers=matchers, + excluders=excluders, + max_depth=max_depth, + hidden=hidden, + follow_symlinks=follow_symlinks, + sort=sort, + on_error=on_error, + visited=visited, + entry_filters=entry_filters, + gitignore=gitignore, + ) + + +def find( + base: str | os.PathLike[str], + patterns: str | Sequence[str] = "*", + *, + exclude: str | Sequence[str] = (), + max_depth: int | None = None, + hidden: bool = False, + follow_symlinks: bool = False, + case_sensitive: bool | None = None, + sort: bool = True, + on_error: OnError = "warn", + kinds: Iterable[Kind] = (), + min_size: int | float | str | None = None, + max_size: int | float | str | None = None, + newer_than: datetime | timedelta | str | None = None, + older_than: datetime | timedelta | str | None = None, + newer_than_file: str | os.PathLike[str] | None = None, + perm: int | str | None = None, + uid: int | None = None, + gid: int | None = None, + respect_gitignore: bool = False, +) -> Iterator[Path]: + """Recursively yield filesystem entries matching ``patterns`` under ``base``. + + Args: + base: Root directory to walk. + patterns: Glob pattern or sequence of patterns. ``"*"`` (the default) + matches everything. Patterns without separators match against + entry basenames; patterns with ``/`` or ``**`` match against the + relative path from ``base``. ``**`` matches any number of path + components, including zero. + exclude: Pattern or patterns to exclude. Same syntax as ``patterns``; + matching entries are skipped *and* not descended into (i.e. + pruning behaviour for directories). + max_depth: Maximum recursion depth. ``None`` means unbounded; ``0`` + limits to direct children of ``base``. + hidden: When ``False`` (default), entries whose name starts with + ``"."`` are skipped (no descent into hidden directories). + follow_symlinks: When ``False`` (default), symbolic links to + directories are *not* descended into. When ``True``, symlink + cycles are terminated by a per-call ``realpath`` memo. + case_sensitive: Force case-sensitive (``True``) or case-insensitive + (``False``) matching. ``None`` (default) follows the host OS + (case-sensitive on Linux, case-insensitive on macOS/Windows). + sort: When ``True`` (default), entries within each directory are + yielded in sorted order, giving a deterministic depth-first + traversal. Set ``False`` for raw ``scandir`` order — this is + the **performance mode** when stable ordering isn't required; + it skips a per-directory sort that costs ~5-10% on large + trees and makes the walker proportionally lighter for + agent/CLI consumers that pipe results through their own + sorter (or don't care about order at all, like ``--null`` + and ``--jsonl`` consumers). + on_error: How to handle ``OSError`` / ``PermissionError`` while + walking. ``"warn"`` (default) emits a :class:`RuntimeWarning`; + ``"ignore"`` swallows the error; ``"raise"`` re-raises. + kinds: Iterable of ``"f"`` (regular file), ``"d"`` (directory), + ``"l"`` (symlink), ``"x"`` (executable). Empty (default) means + "all kinds". + min_size: Minimum file size as bytes (``int``/``float``) or string + ("1K", "5MiB", "100 MB"). Directories and symlinks bypass this + filter. + max_size: Maximum file size — same format as ``min_size``. + newer_than: Only yield entries with mtime strictly after this + timestamp. Accepts :class:`datetime`, :class:`timedelta` + (interpreted as "now minus delta"), an ISO date string, or a + relative duration like ``"7d"`` / ``"3h"`` / ``"2w"``. + older_than: Only yield entries with mtime strictly before this + timestamp — same format as ``newer_than``. + newer_than_file: Only yield entries with mtime strictly after this + file's mtime. + perm: POSIX permission filter. Plain values require exact mode, + ``-MODE`` requires all bits, and ``/MODE`` requires any bit. + uid: POSIX owner uid filter. + gid: POSIX owner gid filter. + respect_gitignore: When ``True``, skip files that any `.gitignore` + under ``base`` would ignore. Requires the optional ``pathspec`` + dependency (``pip install rglob[gitignore]``); silently + no-ops if not installed. + + Yields: + :class:`pathlib.Path` instances for each match, in deterministic + order when ``sort=True``. + + Examples: + >>> from rglob import find + >>> list(find("/tmp", "*.txt")) # doctest: +SKIP + >>> list(find("/tmp", ["*.py", "*.pyx"])) # doctest: +SKIP + >>> list(find(".", "**/test_*.py")) # doctest: +SKIP + """ + base_path = Path(base) + if isinstance(patterns, str): + patterns = (patterns,) + if isinstance(exclude, str): + exclude = (exclude,) + if case_sensitive is None: + case_sensitive = _os_default_case_sensitive() + + matchers = _compile_patterns(patterns, case_sensitive=case_sensitive) + excluders = _compile_patterns(exclude, case_sensitive=case_sensitive) + handler = _make_error_handler(on_error) + visited: set[str] = {os.path.realpath(base_path)} if follow_symlinks else set() + + if newer_than_file is not None: + newer_stat = Path(newer_than_file).stat() + newer_than = datetime.fromtimestamp(newer_stat.st_mtime, tz=UTC) + + entry_filters: list[Callable[[os.DirEntry[str]], bool]] = [] + kinds_set = frozenset(kinds) + if kinds_set: + entry_filters.append(kinds_predicate(kinds_set)) + if min_size is not None or max_size is not None: + entry_filters.append(size_predicate(min_size, max_size)) + if newer_than is not None or older_than is not None: + entry_filters.append(mtime_predicate(newer_than, older_than)) + if perm is not None: + entry_filters.append(perm_predicate(perm)) + if uid is not None or gid is not None: + entry_filters.append(owner_predicate(uid, gid)) + + gitignore_check = gitignore_matcher(base_path) if respect_gitignore else None + + return _walk( + base_path, + "", + 0, + matchers=matchers, + excluders=excluders, + max_depth=max_depth, + hidden=hidden, + follow_symlinks=follow_symlinks, + sort=sort, + on_error=handler, + visited=visited, + entry_filters=tuple(entry_filters), + gitignore=gitignore_check, + ) + + +def find_all( + base: str | os.PathLike[str], + patterns: str | Sequence[str] = "*", + **kwargs: object, +) -> list[Path]: + """Eager variant of :func:`find` — returns ``list[Path]``.""" + return list(find(base, patterns, **kwargs)) # type: ignore[arg-type] + + +# ─── Legacy API (Phase 6 flips rglob/rglob_ to list[Path]) ──────────────────── + + +def rglob(base: str | os.PathLike[str], pattern: str) -> list[Path]: + """Recursively glob entries under ``base`` matching ``pattern``. + + **Breaking change in 2.0**: now returns ``list[Path]`` (was ``list[str]`` + in 1.x). See the migration guide and ADR-0003 for context. To restore + the old behaviour locally: + + paths = [str(p) for p in rglob(base, pattern)] + """ + return list(find(base, pattern, sort=False, on_error="ignore")) + + +def rglob_(pattern: str) -> list[Path]: + """Recursively glob entries under the current working directory.""" + return rglob(str(Path.cwd()), pattern) + + +def _count(files: Iterable[Path | str], func: Callable[[str], bool]) -> int: + """Count lines in ``files`` that satisfy ``func``.""" + total = 0 + for path in files: + with Path(path).open("r", encoding="utf-8", errors="ignore") as handle: + total += sum(1 for line in handle if func(line)) + return total + + +def _sum_sizes(files: Iterable[Path | str]) -> int: + """Sum sizes of file entries (directories are skipped).""" + total = 0 + for path in files: + p = Path(path) + if not p.is_dir(): + total += p.stat().st_size + return total + + +def lcount( + base: str | os.PathLike[str], + pattern: str, + func: Callable[[str], bool] = lambda _line: True, +) -> int: + """Count lines across files matching ``pattern`` under ``base``. + + Args: + base: root directory to walk. + pattern: glob pattern (e.g. ``"*.py"``). + func: per-line predicate; only lines for which it returns truthy + are counted. Defaults to "count every line". + + Returns: + Integer total line count. + """ + return _count(rglob(base, pattern), func) + + +def tsize( + base: str | os.PathLike[str], + pattern: str, + func: Callable[[float], float] = megabytes, +) -> float: + """Sum the total size of files matching ``pattern`` under ``base``. + + Args: + base: root directory to walk. + pattern: glob pattern (e.g. ``"*.jpg"``). + func: unit-conversion function from bytes to the desired unit. + Defaults to :func:`megabytes`. + + Returns: + The total size converted by ``func`` (a float). + """ + return func(_sum_sizes(rglob(base, pattern))) + + +if __name__ == "__main__": # pragma: no cover - manual smoke entry + + def _filter(line: str) -> bool: + return bool(line.strip()) and not line.strip().startswith("#") + + here = Path(__file__).parent + print(f"{lcount(str(here), '*.py', _filter)} non-blank/non-comment lines") diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/__snapshots__/test_cli.ambr b/tests/__snapshots__/test_cli.ambr new file mode 100644 index 0000000..e34290b --- /dev/null +++ b/tests/__snapshots__/test_cli.ambr @@ -0,0 +1,21 @@ +# serializer version: 1 +# name: test_find_default_output + ''' + /a.py + /b.py + /sub/nested.py + + ''' +# --- +# name: test_lcount_total + ''' + Total lines: 8 + + ''' +# --- +# name: test_tsize_default_unit + ''' + Total size: 0.00 MB + + ''' +# --- diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..03b0aed --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,134 @@ +"""Shared pytest fixtures and helpers for rglob tests. + +These mirror the Behave step helpers in `features/steps/steps.py` so the BDD +suite and the pytest suite share the same notion of "build a tree, count +things in it." The fixture style favours `tmp_path` so each test gets an +isolated tree. +""" + +from __future__ import annotations + +import os +import tempfile +from collections.abc import Iterator +from pathlib import Path + +import pytest + + +@pytest.fixture +def tree_builder(tmp_path: Path) -> Iterator[TreeBuilder]: + """Yield a TreeBuilder rooted at tmp_path.""" + builder = TreeBuilder(tmp_path) + yield builder + + +class TreeBuilder: + """Build synthetic directory trees for tests. + + Mirrors the Behave `context.root` / `context.dirs` / `context.known_sizes` + bookkeeping pattern so test logic ports 1:1 from the BDD suite. + """ + + def __init__(self, root: Path) -> None: + """Initialise builder with an empty root directory.""" + self.root: Path = root + self.dirs: list[Path] = [] + self.known_sizes: dict[str, list[int]] = {} + + def make_subdirs(self, n: int) -> list[Path]: + """Create `n` subdirectories under every existing dir (or root if none).""" + targets = list(self.dirs) if self.dirs else [self.root] + created: list[Path] = [] + for parent in targets: + for _ in range(n): + sub = Path(tempfile.mkdtemp(prefix="subdir_", suffix="_rglob", dir=str(parent))) + created.append(sub) + self.dirs.extend(created) + return created + + def make_files(self, n: int, ext: str, *, content: str | None = None) -> list[Path]: + """Create `n` files with extension `ext` under every subdirectory.""" + created: list[Path] = [] + self.known_sizes.setdefault(ext, []) + for d in self.dirs: + for i in range(n): + payload = content if content is not None else f"{i}: synthetic" + path = Path(tempfile.mktemp(prefix="rglob_test_file", suffix=ext, dir=str(d))) + path.write_text(payload, encoding="utf-8") + self.known_sizes[ext].append(path.stat().st_size) + created.append(path) + return created + + def write_lines(self, ext: str, lines: int) -> None: + """Overwrite every `*{ext}` file with `lines` lines of content.""" + for d in self.dirs: + for p in d.iterdir(): + if p.suffix == ext: + p.write_text( + "".join(f"line {i + 1}\n" for i in range(lines)), + encoding="utf-8", + ) + + def total_known_size(self, ext: str) -> int: + """Sum the recorded sizes for files with the given extension.""" + return sum(self.known_sizes.get(ext, [])) + + +@pytest.fixture +def chdir(monkeypatch: pytest.MonkeyPatch) -> object: + """Return a helper to chdir using monkeypatch (auto-reverted).""" + + def _chdir(target: Path) -> None: + monkeypatch.chdir(target) + + return _chdir + + +@pytest.fixture +def make_symlink_loop(tmp_path: Path) -> Path: + """Create `a → b → a` symlink loop and return the entry point. + + Skips on platforms where symlink creation requires elevation (Windows + without developer mode). + """ + a = tmp_path / "a" + b = tmp_path / "b" + a.mkdir() + b.mkdir() + try: + (a / "to_b").symlink_to(b, target_is_directory=True) + (b / "to_a").symlink_to(a, target_is_directory=True) + except (OSError, NotImplementedError) as e: # pragma: no cover - Windows-only + pytest.skip(f"Symlink creation not permitted: {e}") + return a + + +@pytest.fixture(autouse=True) +def _isolate_cwd(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Chdir to tmp_path by default so `rglob_()` tests don't escape isolation.""" + monkeypatch.chdir(tmp_path) + _ = os.getcwd # Touch to silence unused-import flake + + +@pytest.fixture +def case_sensitive_fs(tmp_path: Path) -> bool: + """Return True iff the filesystem under tmp_path distinguishes file casing. + + APFS (macOS default) and NTFS (Windows default) treat ``README.MD`` and + ``readme.md`` as the same path; tests that assume both can coexist as + separate inodes must guard themselves with this fixture. + """ + lower = tmp_path / "__case_probe" + upper = tmp_path / "__CASE_PROBE" + lower.write_text("lower") + try: + upper.write_text("upper") + except OSError: + lower.unlink(missing_ok=True) + return False + try: + return not lower.samefile(upper) + finally: + lower.unlink(missing_ok=True) + upper.unlink(missing_ok=True) diff --git a/tests/fixtures/agent-tree/.gitignore b/tests/fixtures/agent-tree/.gitignore new file mode 100644 index 0000000..de475e5 --- /dev/null +++ b/tests/fixtures/agent-tree/.gitignore @@ -0,0 +1,2 @@ +ignored.log +ignored/ diff --git a/tests/fixtures/agent-tree/binary.bin b/tests/fixtures/agent-tree/binary.bin new file mode 100644 index 0000000..f7cf6e5 Binary files /dev/null and b/tests/fixtures/agent-tree/binary.bin differ diff --git a/tests/fixtures/agent-tree/docs/notes.txt b/tests/fixtures/agent-tree/docs/notes.txt new file mode 100644 index 0000000..5ed57e1 --- /dev/null +++ b/tests/fixtures/agent-tree/docs/notes.txt @@ -0,0 +1,3 @@ +Agent fixture notes. + +TODO: exercise grep context capture. diff --git a/tests/fixtures/agent-tree/duplicates/a.txt b/tests/fixtures/agent-tree/duplicates/a.txt new file mode 100644 index 0000000..d2aa2d8 --- /dev/null +++ b/tests/fixtures/agent-tree/duplicates/a.txt @@ -0,0 +1 @@ +duplicate payload diff --git a/tests/fixtures/agent-tree/duplicates/b.txt b/tests/fixtures/agent-tree/duplicates/b.txt new file mode 100644 index 0000000..d2aa2d8 --- /dev/null +++ b/tests/fixtures/agent-tree/duplicates/b.txt @@ -0,0 +1 @@ +duplicate payload diff --git a/tests/fixtures/agent-tree/hidden/.secret.py b/tests/fixtures/agent-tree/hidden/.secret.py new file mode 100644 index 0000000..ca55820 --- /dev/null +++ b/tests/fixtures/agent-tree/hidden/.secret.py @@ -0,0 +1 @@ +SECRET = "hidden fixture" diff --git a/tests/fixtures/agent-tree/src/main.py b/tests/fixtures/agent-tree/src/main.py new file mode 100644 index 0000000..04f95eb --- /dev/null +++ b/tests/fixtures/agent-tree/src/main.py @@ -0,0 +1,12 @@ +"""Fixture module for agent contract tests.""" + +from src.utils.helper import helper + + +def main() -> str: + """Return the fixture greeting.""" + return helper() + + +if __name__ == "__main__": + print(main()) diff --git a/tests/fixtures/agent-tree/src/utils/helper.py b/tests/fixtures/agent-tree/src/utils/helper.py new file mode 100644 index 0000000..7781ae9 --- /dev/null +++ b/tests/fixtures/agent-tree/src/utils/helper.py @@ -0,0 +1,7 @@ +"""Helper fixture module.""" + + +def helper() -> str: + """Return a deterministic fixture value.""" + # TODO: keep this marker for grep fixture coverage. + return "hello from agent fixture" diff --git a/tests/fixtures/agent-tree/unreadable/.keep b/tests/fixtures/agent-tree/unreadable/.keep new file mode 100644 index 0000000..1a9f1b8 --- /dev/null +++ b/tests/fixtures/agent-tree/unreadable/.keep @@ -0,0 +1 @@ +This directory is chmodded or mocked as unreadable by tests. diff --git a/tests/test_agent_api.py b/tests/test_agent_api.py new file mode 100644 index 0000000..9c2468d --- /dev/null +++ b/tests/test_agent_api.py @@ -0,0 +1,317 @@ +"""Tests for the public `rglob.agent` API.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from rglob.agent import ( + CountOptions, + ErrorCode, + FileMatch, + GrepOptions, + WalkOptions, + __agent_api_version__, + all_schemas, + count, + find_duplicates, + grep, + grep_all, + schema_for, + search, + search_all, +) + + +def test_agent_exports_version_and_types(): + """The stable agent namespace exposes versioned contract members.""" + assert __agent_api_version__ == "1.0" + assert FileMatch.__name__ == "FileMatch" + assert schema_for("walk_options")["title"] == "WalkOptions" + assert "walk_options" in all_schemas() + + +def test_search_and_search_all(tmp_path): + """Agent search helpers return FileMatch records.""" + (tmp_path / "a.py").write_text("", encoding="utf-8") + (tmp_path / "b.txt").write_text("", encoding="utf-8") + + opts = WalkOptions(patterns=["*.py"], base=tmp_path) + result = search_all(opts) + streamed = list(search(opts)) + + assert result.results[0].relative_path == "a.py" + assert streamed[0].relative_path == "a.py" + + +def test_search_all_converts_bad_predicate_to_error(tmp_path): + """Operational predicate errors become ErrorInfo records.""" + result = search_all(WalkOptions(patterns=["*"], base=tmp_path, perm="nope")) + assert result.results == [] + assert result.errors[0].code == ErrorCode.BAD_PREDICATE + + +def test_search_all_strict_base_rejects_symlink_escape(tmp_path): + """Default strict_base skips symlinks resolving outside the base.""" + base = tmp_path / "base" + base.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + target = outside / "secret.py" + target.write_text("", encoding="utf-8") + try: + (base / "secret.py").symlink_to(target) + except (OSError, NotImplementedError): # pragma: no cover - Windows-only + pytest.skip("symlinks not permitted") + + result = search_all(WalkOptions(patterns=["*.py"], base=base)) + + assert result.results == [] + assert result.errors[0].code == ErrorCode.PERM + + +def test_search_all_timeout_is_truncated(monkeypatch, tmp_path): + """timeout_seconds is honored by the public search API.""" + (tmp_path / "a.py").write_text("", encoding="utf-8") + ticks = iter([0.0, 1.0]) + monkeypatch.setattr("rglob.agent._runtime.time.monotonic", lambda: next(ticks)) + + result = search_all(WalkOptions(patterns=["*.py"], base=tmp_path, timeout_seconds=0.1)) + + assert result.truncated is True + assert result.truncated_reason == "timeout" + assert result.errors[0].code == ErrorCode.TIMEOUT + + +def test_grep_and_grep_all(tmp_path): + """Agent grep helpers return LineMatch records.""" + (tmp_path / "notes.txt").write_text("TODO\n", encoding="utf-8") + + opts = GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path) + result = grep_all(opts) + streamed = list(grep(opts)) + + assert result.results[0].content == "TODO" + assert streamed[0].line_number == 1 + + +def test_grep_all_timeout_is_error(monkeypatch, tmp_path): + """timeout_seconds is honored by the public grep API.""" + (tmp_path / "notes.txt").write_text("TODO\n", encoding="utf-8") + ticks = iter([0.0, 1.0]) + monkeypatch.setattr("rglob.agent._runtime.time.monotonic", lambda: next(ticks)) + + result = grep_all( + GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path, timeout_seconds=0.1) + ) + + assert result.truncated is True + assert result.truncated_reason == "timeout" + assert result.errors[0].code == ErrorCode.TIMEOUT + + +def test_grep_all_converts_operational_error(monkeypatch, tmp_path): + """Unexpected grep errors are converted into LineSearchResult errors.""" + + def boom(_opts): + raise ValueError("bad") + + monkeypatch.setattr("rglob._grep.grep_all", boom) + result = grep_all(GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path)) + assert result.errors[0].code == ErrorCode.BAD_PREDICATE + + +def test_count_and_count_error(tmp_path): + """Agent count returns Stats and converts bad predicates.""" + (tmp_path / "a.py").write_text("x\n", encoding="utf-8") + + good = count(CountOptions(patterns=["*.py"], base=tmp_path)) + bad = count(CountOptions(patterns=["*.py"], base=tmp_path, newer_than_file=Path("missing"))) + + assert good.files == 1 + assert good.lines == 1 + assert bad.errors[0].code == ErrorCode.UNREADABLE + + +def test_count_timeout_is_error(monkeypatch, tmp_path): + """timeout_seconds is honored by the public count API.""" + (tmp_path / "a.py").write_text("x\n", encoding="utf-8") + ticks = iter([0.0, 1.0]) + monkeypatch.setattr("rglob.agent._runtime.time.monotonic", lambda: next(ticks)) + + result = count(CountOptions(patterns=["*.py"], base=tmp_path, timeout_seconds=0.1)) + + assert result.truncated is True + assert result.truncated_reason == "timeout" + assert result.errors[0].code == ErrorCode.TIMEOUT + + +def test_count_converts_permission_error(monkeypatch, tmp_path): + """Permission errors map to the PERM error code.""" + + def boom(_opts): + raise PermissionError("denied") + + monkeypatch.setattr("rglob._count.count_all", boom) + result = count(CountOptions(patterns=["*.py"], base=tmp_path)) + assert result.errors[0].code == ErrorCode.PERM + + +def test_find_duplicates_agent_result(tmp_path): + """Duplicate API returns structured duplicate groups.""" + payload = b"same" + (tmp_path / "a.bin").write_bytes(payload) + (tmp_path / "b.bin").write_bytes(payload) + (tmp_path / "c.bin").write_bytes(b"other") + + result = find_duplicates(WalkOptions(patterns=["*.bin"], base=tmp_path, limit=1)) + + assert len(result.results) == 1 + assert {path.name for path in result.results[0].paths} == {"a.bin", "b.bin"} + + +def test_find_duplicates_limit_reports_truncation(tmp_path): + """Duplicate result limits set truncation metadata.""" + (tmp_path / "a.bin").write_bytes(b"one") + (tmp_path / "b.bin").write_bytes(b"one") + (tmp_path / "c.bin").write_bytes(b"two") + (tmp_path / "d.bin").write_bytes(b"two") + + result = find_duplicates(WalkOptions(patterns=["*.bin"], base=tmp_path, limit=1)) + + assert len(result.results) == 1 + assert result.truncated is True + assert result.truncated_reason == "limit" + + +def test_find_duplicates_handles_stat_error_in_record(monkeypatch, tmp_path): + """Duplicate records fall back to size 0 when stat fails.""" + missing_a = tmp_path / "missing-a.bin" + missing_b = tmp_path / "missing-b.bin" + + def fake_groups(_paths, *, timeout_check=None): + return [[missing_a, missing_b]] + + monkeypatch.setattr("rglob._dupes.find_duplicates", fake_groups) + result = find_duplicates(WalkOptions(patterns=["*.bin"], base=tmp_path)) + + assert result.results[0].size == 0 + + +def test_find_duplicates_handles_empty_group(monkeypatch, tmp_path): + """A defensive empty duplicate group still serializes.""" + + def fake_groups(_paths, *, timeout_check=None): + return [[]] + + monkeypatch.setattr("rglob._dupes.find_duplicates", fake_groups) + result = find_duplicates(WalkOptions(patterns=["*.bin"], base=tmp_path)) + + assert result.results[0].paths == [] + assert result.results[0].size == 0 + + +def test_find_duplicates_converts_errors(tmp_path): + """Duplicate API is non-raising for bad predicates.""" + result = find_duplicates(WalkOptions(patterns=["*"], base=tmp_path, perm="nope")) + assert result.errors[0].code == ErrorCode.BAD_PREDICATE + + +def test_find_duplicates_strict_base_rejects_symlink_escape(tmp_path): + """Duplicate search does not hash linked dirs that resolve outside strict base.""" + base = tmp_path / "base" + base.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + target = outside / "secret.bin" + target.write_bytes(b"same") + try: + (base / "linked").symlink_to(outside, target_is_directory=True) + except (OSError, NotImplementedError): # pragma: no cover - Windows-only + pytest.skip("symlinks not permitted") + + result = find_duplicates(WalkOptions(patterns=["*.bin"], base=base, follow_symlinks=True)) + + assert result.results == [] + assert result.errors[0].code == ErrorCode.PERM + + +def test_find_duplicates_strict_base_can_suppress_error(tmp_path): + """Duplicate strict-base errors respect include_errors=False.""" + base = tmp_path / "base" + base.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + target = outside / "secret.bin" + target.write_bytes(b"same") + try: + (base / "linked").symlink_to(outside, target_is_directory=True) + except (OSError, NotImplementedError): # pragma: no cover - Windows-only + pytest.skip("symlinks not permitted") + + result = find_duplicates( + WalkOptions( + patterns=["*.bin"], + base=base, + follow_symlinks=True, + include_errors=False, + ) + ) + + assert result.results == [] + assert result.errors == [] + + +def test_find_duplicates_timeout_is_error(monkeypatch, tmp_path): + """timeout_seconds is honored by duplicate search.""" + (tmp_path / "a.bin").write_bytes(b"same") + ticks = iter([0.0, 1.0]) + monkeypatch.setattr("rglob.agent._runtime.time.monotonic", lambda: next(ticks)) + + result = find_duplicates(WalkOptions(patterns=["*.bin"], base=tmp_path, timeout_seconds=0.1)) + + assert result.truncated is True + assert result.truncated_reason == "timeout" + assert result.errors[0].code == ErrorCode.TIMEOUT + + +def test_find_duplicates_timeout_during_hashing(monkeypatch, tmp_path): + """Duplicate hashing receives the public timeout deadline.""" + (tmp_path / "a.bin").write_bytes(b"same") + ticks = iter([0.0, 0.0, 1.0]) + monkeypatch.setattr("rglob.agent._runtime.time.monotonic", lambda: next(ticks)) + + def fake_groups(_paths, *, timeout_check=None): + assert timeout_check is not None + timeout_check() + return [] + + monkeypatch.setattr("rglob._dupes.find_duplicates", fake_groups) + + result = find_duplicates(WalkOptions(patterns=["*.bin"], base=tmp_path, timeout_seconds=0.1)) + + assert result.truncated is True + assert result.truncated_reason == "timeout" + assert result.total_files_searched == 1 + assert result.errors[0].code == ErrorCode.TIMEOUT + + +def test_find_duplicates_timeout_can_suppress_error(monkeypatch, tmp_path): + """Duplicate timeout errors respect include_errors=False.""" + (tmp_path / "a.bin").write_bytes(b"same") + ticks = iter([0.0, 1.0]) + monkeypatch.setattr("rglob.agent._runtime.time.monotonic", lambda: next(ticks)) + + result = find_duplicates( + WalkOptions( + patterns=["*.bin"], + base=tmp_path, + timeout_seconds=0.1, + include_errors=False, + ) + ) + + assert result.truncated is True + assert result.truncated_reason == "timeout" + assert result.errors == [] diff --git a/tests/test_agent_contract.py b/tests/test_agent_contract.py new file mode 100644 index 0000000..a062eca --- /dev/null +++ b/tests/test_agent_contract.py @@ -0,0 +1,285 @@ +"""Tests for the agent contract models and golden fixture.""" + +from __future__ import annotations + +import os +from dataclasses import FrozenInstanceError, is_dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest + +from rglob.agent import all_schemas, schema_for +from rglob.agent._introspection import _schema_for_type +from rglob.agent._models import ( + CapabilityReport, + ErrorCode, + ErrorInfo, + FileMatch, + FileSearchResult, + error_envelope, + to_json_dict, +) +from rglob.agent._runtime import _strict_base_error, collect_file_search, file_match + +FIXTURE = Path(__file__).parent / "fixtures" / "agent-tree" + + +def test_agent_fixture_shape(): + """The committed golden fixture contains all required test surfaces.""" + required = { + ".gitignore", + "src/main.py", + "src/utils/helper.py", + "docs/notes.txt", + "binary.bin", + "unreadable/.keep", + "duplicates/a.txt", + "duplicates/b.txt", + "hidden/.secret.py", + } + assert {path.as_posix() for path in required_paths(FIXTURE)} >= required + assert (FIXTURE / "duplicates" / "a.txt").read_bytes() == ( + FIXTURE / "duplicates" / "b.txt" + ).read_bytes() + with pytest.raises(UnicodeDecodeError): + (FIXTURE / "binary.bin").read_text(encoding="utf-8") + + +def required_paths(root: Path) -> set[Path]: + """Return fixture paths relative to root.""" + return {path.relative_to(root) for path in root.rglob("*") if path.is_file()} + + +def test_file_match_is_frozen_and_slotted(): + """Contract dataclasses are frozen, slotted dataclasses.""" + match = FileMatch( + path=FIXTURE / "src" / "main.py", + relative_path="src/main.py", + size=10, + mtime=datetime(2026, 5, 15, tzinfo=UTC), + kinds=["f"], + errors=[], + ) + + assert is_dataclass(match) + assert not hasattr(match, "__dict__") + with pytest.raises(FrozenInstanceError): + match.size = 11 # type: ignore[misc] + + +def test_to_json_dict_serializes_wire_types(): + """Path, datetime, StrEnum, and nested dataclasses become JSON-safe.""" + error = ErrorInfo(ErrorCode.UNREADABLE, "cannot read", FIXTURE / "unreadable") + result = FileSearchResult( + results=[ + FileMatch( + path=FIXTURE / "src" / "main.py", + relative_path="src/main.py", + size=123, + mtime=datetime(2026, 5, 15, 12, 30, tzinfo=UTC), + kinds=["f"], + errors=[error], + ) + ], + truncated=False, + total_files_searched=1, + bytes_read=123, + errors=[error], + truncated_reason=None, + ) + + payload = to_json_dict(result) + + assert isinstance(payload, dict) + # to_json_dict serialises Path via str(), which uses the host's native + # separator — so the suffix probe must use the same separator. + assert payload["results"][0]["path"].endswith(str(Path("src") / "main.py")) + assert payload["results"][0]["mtime"] == "2026-05-15T12:30:00Z" + assert payload["errors"][0]["code"] == "UNREADABLE" + assert to_json_dict(object()).startswith("= { + "file_search_result", + "line_search_result", + "duplicate_search_result", + "walk_options", + "grep_options", + "count_options", + } + for payload in schemas.values(): + assert payload["$schema"] == "https://json-schema.org/draft/2020-12/schema" + assert payload["version"] == "1.0" + + walk = schema_for("walk_options") + assert walk == schemas["walk_options"] + assert walk["title"] == "WalkOptions" + assert walk["properties"]["base"]["format"] == "path" + + +def test_schema_generator_handles_open_ended_types(): + """Fallback schema branches remain deterministic.""" + assert _schema_for_type(Any) == {} + assert _schema_for_type(object) == {"type": "string"} + with pytest.raises(ValueError, match="unknown schema: missing"): + schema_for("missing") + + +def test_file_match_handles_paths_outside_base(tmp_path): + """Relative paths fall back to the basename when strict base is not applied.""" + outside = tmp_path / "outside.txt" + base = tmp_path / "base" + base.mkdir() + outside.write_text("outside") + + match = file_match(outside, base, include_errors=True) + + assert match.relative_path == "outside.txt" + + +def test_file_match_reports_dir_symlink_and_executable_kinds(tmp_path): + """FileMatch kind detection covers dirs, symlinks, files, and executables.""" + directory = tmp_path / "dir" + directory.mkdir() + executable = tmp_path / "run.sh" + executable.write_text("#!/bin/sh\n") + executable.chmod(0o755) + + dir_match = file_match(directory, tmp_path, include_errors=True) + exe_match = file_match(executable, tmp_path, include_errors=True) + + assert "d" in dir_match.kinds + assert "f" in exe_match.kinds + if os.name == "posix": + # NTFS does not honour Path.chmod() for executable bits, so the + # `x` tag is only reliably present on POSIX hosts. + assert "x" in exe_match.kinds + + try: + link = tmp_path / "run-link.sh" + link.symlink_to(executable) + except (OSError, NotImplementedError): # pragma: no cover - Windows-only + pytest.skip("symlinks not permitted") + + link_match = file_match(link, tmp_path, include_errors=True) + assert "l" in link_match.kinds + + +def test_file_match_can_suppress_stat_errors(tmp_path): + """include_errors=False keeps OSErrors out of the match record.""" + match = file_match(tmp_path / "missing.txt", tmp_path, include_errors=False) + assert match.size == 0 + assert match.errors == [] + + +def test_collect_file_search_max_bytes_truncates(tmp_path): + """max_bytes truncates before emitting a record that would exceed it.""" + path = tmp_path / "a.txt" + path.write_text("hello") + + result = collect_file_search( + [path], + base=tmp_path, + limit=None, + max_bytes=1, + include_errors=True, + ) + + assert result.results == [] + assert result.truncated is True + assert result.truncated_reason == "max_bytes" + + +def test_collect_file_search_timeout_can_suppress_error(monkeypatch, tmp_path): + """Timeout truncation respects include_errors=False.""" + path = tmp_path / "a.txt" + path.write_text("hello") + ticks = iter([0.0, 1.0]) + monkeypatch.setattr("rglob.agent._runtime.time.monotonic", lambda: next(ticks)) + + result = collect_file_search( + [path], + base=tmp_path, + limit=None, + max_bytes=None, + include_errors=False, + timeout_seconds=0.1, + ) + + assert result.truncated_reason == "timeout" + assert result.errors == [] + + +def test_collect_file_search_strict_base_can_suppress_error(tmp_path): + """Strict-base containment skips escaped paths without errors when requested.""" + outside = tmp_path / "outside.txt" + outside.write_text("hello") + base = tmp_path / "base" + base.mkdir() + + result = collect_file_search( + [outside], + base=base, + limit=None, + max_bytes=None, + include_errors=False, + strict_base=True, + ) + + assert result.results == [] + assert result.errors == [] + + +def test_timeout_floor_is_bad_predicate(tmp_path): + """Timeouts below the supported floor are rejected by the agent API.""" + from rglob.agent import WalkOptions, search_all + + result = search_all(WalkOptions(patterns=["*"], base=tmp_path, timeout_seconds=0.01)) + assert result.errors[0].code == ErrorCode.BAD_PREDICATE + + +def test_strict_base_error_accepts_relative_path(): + """The strict-base error helper handles relative paths lexically.""" + error = _strict_base_error(Path("outside.txt"), Path.cwd()) + assert error.path == Path.cwd() / "outside.txt" diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..bc68871 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,723 @@ +"""Tests for the Typer-powered CLI (Phase 4). + +Uses :class:`typer.testing.CliRunner` for in-process invocation and +:func:`syrupy.snapshot` for golden-file assertions on stdout / stderr. + +Path-stripping helpers normalise the snapshots so tmp_path doesn't leak +into them. +""" + +from __future__ import annotations + +import json +import re +import subprocess +import sys +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from rglob.cli import app + +# Use a wide virtual terminal so Rich doesn't truncate paths in test output. +runner = CliRunner(env={"COLUMNS": "200"}) + + +def _build_tree(root): + """Build a small fixture tree under `root`.""" + (root / "a.py").write_text("alpha\n# comment\n\nbeta\n") + (root / "b.py").write_text("x\ny\nz\n") + (root / "c.txt").write_text("hello\n") + sub = root / "sub" + sub.mkdir() + (sub / "nested.py").write_text("nested\n") + hidden = root / ".hidden" + hidden.mkdir() + (hidden / "secret.py").write_text("secret\n") + + +def _normalise(output: str, base) -> str: + """Strip the absolute tmp_path so snapshots are portable across OSes.""" + sanitized = output.replace(str(base), "") + # Remove tmp_path leftovers in nested paths (Linux-only). + sanitized = re.sub(r"/tmp/pytest-of-[a-zA-Z0-9_]+/pytest-\d+/[^/\s]+", "", sanitized) + # Canonicalise to forward slashes so Windows CLI output matches the + # snapshot recorded on Linux. + return sanitized.replace("\\", "/") + + +# ─── find ──────────────────────────────────────────────────────────────────── + + +def test_find_default_output(tmp_path, snapshot): + """`rglob find` lists paths one per line by default.""" + _build_tree(tmp_path) + result = runner.invoke(app, ["find", "*.py", "--base", str(tmp_path)]) + assert result.exit_code == 0 + assert _normalise(result.stdout, tmp_path) == snapshot + + +def test_find_json_array(tmp_path): + """`--json` emits a FileSearchResult object.""" + _build_tree(tmp_path) + result = runner.invoke(app, ["find", "*.py", "--base", str(tmp_path), "--json"]) + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert set(payload) >= { + "results", + "truncated", + "total_files_searched", + "bytes_read", + "errors", + "truncated_reason", + } + names = {Path(match["path"]).name for match in payload["results"]} + assert names == {"a.py", "b.py", "nested.py"} + assert payload["truncated"] is False + + +def test_find_jsonl(tmp_path): + """`--jsonl` emits one compact FileSearchResult object line.""" + _build_tree(tmp_path) + result = runner.invoke(app, ["find", "*.py", "--base", str(tmp_path), "--jsonl"]) + assert result.exit_code == 0 + lines = [line for line in result.stdout.splitlines() if line.strip()] + assert len(lines) == 1 + payload = json.loads(lines[0]) + assert "results" in payload + assert {Path(match["path"]).name for match in payload["results"]} == { + "a.py", + "b.py", + "nested.py", + } + + +def test_find_null_separator(tmp_path): + """`-0` separates paths with NUL bytes for `xargs -0`.""" + _build_tree(tmp_path) + result = runner.invoke(app, ["find", "*.py", "--base", str(tmp_path), "-0"]) + assert result.exit_code == 0 + assert "\x00" in result.stdout + + +def test_find_format_template(tmp_path): + """`--format` interpolates per-path fields.""" + _build_tree(tmp_path) + result = runner.invoke( + app, + [ + "find", + "*.py", + "--base", + str(tmp_path), + "--format", + "{name}|{size}", + ], + ) + assert result.exit_code == 0 + for line in result.stdout.splitlines(): + name, _, size = line.partition("|") + assert name.endswith(".py") + assert size.isdigit() + + +def test_find_exclude(tmp_path): + """`--exclude` prunes the directory.""" + _build_tree(tmp_path) + result = runner.invoke( + app, + ["find", "*.py", "--base", str(tmp_path), "-E", "sub"], + ) + assert result.exit_code == 0 + assert "nested.py" not in result.stdout + assert "a.py" in result.stdout + + +def test_find_max_depth(tmp_path): + """`--max-depth 0` keeps the walk at the base level.""" + _build_tree(tmp_path) + result = runner.invoke( + app, + ["find", "*.py", "--base", str(tmp_path), "-d", "0"], + ) + assert result.exit_code == 0 + assert "nested.py" not in result.stdout + + +def test_find_hidden(tmp_path): + """`--hidden/-H` reveals dot-directory contents.""" + _build_tree(tmp_path) + result = runner.invoke( + app, + ["find", "*.py", "--base", str(tmp_path), "-H"], + ) + assert result.exit_code == 0 + assert "secret.py" in result.stdout + + +def test_find_multiple_patterns(tmp_path): + """Multiple positional patterns are OR'd.""" + _build_tree(tmp_path) + result = runner.invoke( + app, + ["find", "*.py", "*.txt", "--base", str(tmp_path)], + ) + assert result.exit_code == 0 + assert "a.py" in result.stdout + assert "c.txt" in result.stdout + + +def test_find_warns_on_pre_expansion(tmp_path): + """Multiple positional args with no glob chars → warning.""" + _build_tree(tmp_path) + result = runner.invoke( + app, + ["find", "a.py", "b.py", "--base", str(tmp_path)], + ) + # Warning currently goes through Rich; verify it surfaces in stderr OR stdout. + combined = (result.stderr or "") + (result.stdout or "") + assert "pre-expand" in combined or result.exit_code == 0 + + +def test_find_json_limit_reports_truncation(tmp_path): + """Structured find output reports truncation metadata.""" + _build_tree(tmp_path) + result = runner.invoke( + app, + ["find", "*.py", "--base", str(tmp_path), "--json", "--limit", "1"], + ) + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert len(payload["results"]) == 1 + assert payload["truncated"] is True + assert payload["truncated_reason"] == "limit" + + +def test_find_json_bad_max_bytes_is_error_envelope(tmp_path): + """Bad structured resource limits produce machine-readable errors.""" + _build_tree(tmp_path) + result = runner.invoke( + app, + ["find", "*.py", "--base", str(tmp_path), "--json", "--max-bytes", "nope"], + ) + assert result.exit_code == 2 + payload = json.loads(result.stdout) + assert payload["ok"] is False + assert payload["error"]["code"] == "BAD_PREDICATE" + + +def test_find_plain_bad_max_bytes_is_human_error(tmp_path): + """Bad human-output resource limits still produce a CLI error.""" + _build_tree(tmp_path) + result = runner.invoke( + app, + ["find", "*.py", "--base", str(tmp_path), "--max-bytes", "nope"], + ) + assert result.exit_code == 2 + assert "unparseable size" in (result.stdout + result.stderr) + + +def test_grep_json_output(tmp_path): + """`rglob grep --json` emits a LineSearchResult.""" + (tmp_path / "notes.txt").write_text("TODO one\nnope\n", encoding="utf-8") + result = runner.invoke( + app, + ["grep", "TODO", "*.txt", "--base", str(tmp_path), "--json"], + ) + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["results"][0]["content"] == "TODO one" + assert payload["results"][0]["line_number"] == 1 + + +def test_grep_jsonl_output(tmp_path): + """`rglob grep --jsonl` streams one match per line then a summary record.""" + (tmp_path / "notes.txt").write_text("TODO one\nTODO two\n", encoding="utf-8") + result = runner.invoke( + app, + ["grep", "TODO", "*.txt", "--base", str(tmp_path), "--jsonl"], + ) + assert result.exit_code == 0 + lines = [json.loads(line) for line in result.stdout.splitlines() if line.strip()] + matches = [item for item in lines if item.get("kind") == "match"] + summaries = [item for item in lines if item.get("kind") == "summary"] + assert len(matches) == 2 + assert {m["content"] for m in matches} == {"TODO one", "TODO two"} + assert len(summaries) == 1 + assert summaries[0]["total_files_searched"] == 1 + assert summaries[0]["truncated"] is False + + +def test_grep_files_with_matches_human_output(tmp_path): + """`rglob grep -l` prints just the matching paths, one per line.""" + (tmp_path / "a.txt").write_text("TODO\n") + (tmp_path / "b.txt").write_text("nope\n") + (tmp_path / "c.txt").write_text("TODO\nTODO\n") + result = runner.invoke( + app, + ["grep", "TODO", "*.txt", "--base", str(tmp_path), "-l"], + ) + assert result.exit_code == 0 + out_lines = sorted(line for line in result.stdout.splitlines() if line.strip()) + # Two paths, one per line, no `:line:content` suffix. + assert len(out_lines) == 2 + assert all(line.endswith(("a.txt", "c.txt")) for line in out_lines) + + +def test_grep_count_only_human_output(tmp_path): + """`rglob grep -c` prints `path:count` lines.""" + (tmp_path / "a.txt").write_text("TODO\nTODO\nTODO\n") + (tmp_path / "b.txt").write_text("none\n") + result = runner.invoke( + app, + ["grep", "TODO", "*.txt", "--base", str(tmp_path), "-c"], + ) + assert result.exit_code == 0 + out_lines = [line for line in result.stdout.splitlines() if line.strip()] + assert any(line.endswith("a.txt:3") for line in out_lines) + + +def test_grep_files_with_matches_and_count_only_are_mutually_exclusive(tmp_path): + """Combining `-l` and `-c` is rejected.""" + (tmp_path / "a.txt").write_text("TODO\n") + result = runner.invoke( + app, + ["grep", "TODO", "*.txt", "--base", str(tmp_path), "-l", "-c"], + ) + assert result.exit_code == 2 + combined = (result.stdout or "") + (result.stderr or "") + assert "mutually exclusive" in combined + + +def test_grep_human_output_and_warning(tmp_path): + """Human grep output prints path:line:content and binary warnings.""" + (tmp_path / "notes.txt").write_text("TODO one\n", encoding="utf-8") + (tmp_path / "binary.bin").write_bytes(b"\x00TODO\x00") + result = runner.invoke(app, ["grep", "TODO", "*", "--base", str(tmp_path)]) + assert result.exit_code == 0 + assert "TODO one" in result.stdout + assert "BINARY" in result.stderr + + +def test_count_jsonl_output(tmp_path): + """`rglob count --jsonl` emits one compact Stats object line.""" + (tmp_path / "a.py").write_text("x\n\n# comment\ny\n", encoding="utf-8") + result = runner.invoke( + app, + ["count", "*.py", "--base", str(tmp_path), "--no-empty", "--no-comments", "--jsonl"], + ) + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["files"] == 1 + assert payload["lines"] == 2 + + +def test_count_json_output(tmp_path): + """`rglob count --json` emits a pretty Stats object.""" + (tmp_path / "a.py").write_text("x\n", encoding="utf-8") + result = runner.invoke(app, ["count", "*.py", "--base", str(tmp_path), "--json"]) + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["files"] == 1 + assert payload["lines"] == 1 + + +def test_count_human_output(tmp_path): + """`rglob count` prints files, lines, and bytes for humans.""" + (tmp_path / "a.py").write_text("x\n", encoding="utf-8") + result = runner.invoke(app, ["count", "*.py", "--base", str(tmp_path)]) + assert result.exit_code == 0 + assert "Files: 1" in result.stdout + assert "Lines: 1" in result.stdout + + +# ─── lcount ────────────────────────────────────────────────────────────────── + + +def test_lcount_total(tmp_path, snapshot): + """`rglob lcount` reports a total.""" + _build_tree(tmp_path) + result = runner.invoke(app, ["lcount", "*.py", "--base", str(tmp_path)]) + assert result.exit_code == 0 + assert _normalise(result.stdout, tmp_path) == snapshot + + +def test_lcount_no_empty_no_comments(tmp_path): + """`--no-empty --no-comments` matches the documented filter behaviour.""" + _build_tree(tmp_path) + # a.py: 4 lines, 2 valid (alpha, beta) + # b.py: 3 lines, all valid + # nested.py: 1 line, valid + result = runner.invoke( + app, + ["lcount", "*.py", "--base", str(tmp_path), "--no-empty", "--no-comments"], + ) + assert result.exit_code == 0 + assert "Total lines: 6" in result.stdout + + +def test_lcount_no_empty_only(tmp_path): + """`--no-empty` keeps comments.""" + _build_tree(tmp_path) + # a.py 3 non-empty (alpha, # comment, beta), b.py 3, nested.py 1 = 7 + result = runner.invoke( + app, + ["lcount", "*.py", "--base", str(tmp_path), "--no-empty"], + ) + assert result.exit_code == 0 + assert "Total lines: 7" in result.stdout + + +# ─── tsize ─────────────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize("unit", ["kb", "mb", "gb", "tb"]) +def test_tsize_units(tmp_path, unit): + """Every documented unit suffix is accepted.""" + (tmp_path / "x.bin").write_bytes(b"x" * 4096) + result = runner.invoke( + app, + ["tsize", "*.bin", "--base", str(tmp_path), "--unit", unit], + ) + assert result.exit_code == 0 + assert unit.upper() in result.stdout + + +def test_tsize_invalid_unit(tmp_path): + """Unknown unit → exit code 2 + helpful message.""" + (tmp_path / "x.bin").write_bytes(b"x") + result = runner.invoke( + app, + ["tsize", "*.bin", "--base", str(tmp_path), "--unit", "petabytes"], + ) + assert result.exit_code == 2 + assert "unknown unit" in (result.stdout + result.stderr).lower() + + +def test_tsize_default_unit(tmp_path, snapshot): + """Default unit is MB; output is float with two decimals.""" + (tmp_path / "x.bin").write_bytes(b"x" * 4096) + result = runner.invoke(app, ["tsize", "*.bin", "--base", str(tmp_path)]) + assert result.exit_code == 0 + assert _normalise(result.stdout, tmp_path) == snapshot + + +# ─── completion / help ─────────────────────────────────────────────────────── + + +def test_help_runs(): + """`rglob --help` returns 0.""" + result = runner.invoke(app, ["--help"]) + assert result.exit_code == 0 + assert "find" in result.stdout + assert "lcount" in result.stdout + assert "tsize" in result.stdout + + +def test_module_help_imports_in_fresh_process(): + """`python -m rglob.cli --help` does not hit agent import cycles.""" + result = subprocess.run( + [sys.executable, "-m", "rglob.cli", "--help"], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert "find" in result.stdout + + +_ANSI_RE = re.compile(r"\x1b\[[0-9;]*[A-Za-z]") + + +def _plain(text: str) -> str: + """Strip ANSI escapes and collapse whitespace for substring assertions. + + Rich's help renderer styles option names and may wrap long ones across + lines when the host terminal is narrow (e.g. on GitHub Actions where + FORCE_COLOR is set). This helper makes substring checks invariant to + both styling and wrapping. + """ + return re.sub(r"\s+", " ", _ANSI_RE.sub("", text)) + + +def test_find_help_lists_filter_flags(): + """`rglob find --help` documents the new filter flags.""" + result = runner.invoke(app, ["find", "--help"]) + assert result.exit_code == 0 + plain = _plain(result.stdout) + for flag in ("--exclude", "--max-depth", "--hidden", "--follow", "--json"): + assert flag in plain, f"expected {flag!r} in help output" + + +# ─── agent introspection ───────────────────────────────────────────────────── + + +def test_describe_find_outputs_manifest_json(): + """`rglob describe find` is a pure JSON manifest.""" + result = runner.invoke(app, ["describe", "find"]) + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["name"] == "find" + assert payload["agent_api_version"] == "1.0" + assert "schemas" in payload + + +def test_schema_find_outputs_input_and_output_schemas(): + """`rglob schema find` exposes the generated schemas.""" + result = runner.invoke(app, ["schema", "find"]) + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["input"]["title"] == "WalkOptions" + assert payload["output"]["title"] == "FileSearchResult" + + +def test_schema_all_outputs_all_public_schemas(): + """`rglob schema --all` exposes every public generated schema.""" + result = runner.invoke(app, ["schema", "--all"]) + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["walk_options"]["title"] == "WalkOptions" + assert payload["grep_options"]["title"] == "GrepOptions" + assert payload["file_search_result"]["title"] == "FileSearchResult" + + +def test_schema_unknown_subcommand_is_error_envelope(): + """Unknown schema targets produce the same stable error envelope.""" + result = runner.invoke(app, ["schema", "missing"]) + assert result.exit_code == 2 + payload = json.loads(result.stdout) + assert payload["ok"] is False + assert payload["error"]["code"] == "BAD_PREDICATE" + + +def test_schema_requires_subcommand_or_all(): + """`rglob schema` without a target stays a JSON error endpoint.""" + result = runner.invoke(app, ["schema"]) + assert result.exit_code == 2 + payload = json.loads(result.stdout) + assert payload["ok"] is False + assert payload["error"]["code"] == "BAD_PREDICATE" + + +def test_capabilities_json_report(): + """`rglob capabilities --json` reports versioned capabilities.""" + result = runner.invoke(app, ["capabilities", "--json"]) + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["agent_api_version"] == "1.0" + assert payload["schema_version"] == "1.0" + assert "extras" in payload + + +def test_agent_version_is_json_string(): + """`rglob agent-version` emits a JSON string.""" + result = runner.invoke(app, ["agent-version"]) + assert result.exit_code == 0 + assert json.loads(result.stdout) == "1.0" + + +def test_describe_unknown_subcommand_is_error_envelope(): + """Unknown introspection targets produce a stable error envelope.""" + result = runner.invoke(app, ["describe", "missing"]) + assert result.exit_code == 2 + payload = json.loads(result.stdout) + assert payload["ok"] is False + assert payload["error"]["code"] == "BAD_PREDICATE" + + +# ─── Coverage edge cases (OSError paths) ───────────────────────────────────── + + +def test_find_format_handles_stat_oserror(tmp_path, monkeypatch): + """`--format` keeps going if a stat() call raises OSError.""" + _build_tree(tmp_path) + real_stat = __import__("pathlib").Path.stat + + def boom(self, *a, **kw): + if self.name == "a.py": + raise PermissionError("no") + return real_stat(self, *a, **kw) + + monkeypatch.setattr("pathlib.Path.stat", boom) + result = runner.invoke( + app, + ["find", "*.py", "--base", str(tmp_path), "--format", "{name}:{size}"], + ) + assert result.exit_code == 0 + assert "a.py:0" in result.stdout # fell back to size=0 + + +def test_find_jsonl_handles_stat_oserror(tmp_path, monkeypatch): + """`--jsonl` records per-match stat errors in the result envelope.""" + _build_tree(tmp_path) + real_stat = __import__("pathlib").Path.stat + + def boom(self, *a, **kw): + if self.name == "a.py": + raise PermissionError("no") + return real_stat(self, *a, **kw) + + monkeypatch.setattr("pathlib.Path.stat", boom) + result = runner.invoke(app, ["find", "*.py", "--base", str(tmp_path), "--jsonl"]) + assert result.exit_code == 0 + payload = json.loads(result.stdout) + a_match = next(obj for obj in payload["results"] if obj["path"].endswith("a.py")) + assert a_match["size"] == 0 + assert a_match["errors"][0]["code"] == "PERM" + + +def test_main_entrypoint_runs(monkeypatch): + """`cli.main()` invokes the Typer app and exits via SystemExit.""" + import sys + + from rglob import cli + + monkeypatch.setattr(sys, "argv", ["rglob", "--help"]) + with pytest.raises(SystemExit) as info: + cli.main() + assert info.value.code == 0 + + +# ─── Phase 5 subcommands ───────────────────────────────────────────────────── + + +def test_stats_subcommand(tmp_path): + """`rglob stats` runs and emits a populated table.""" + _build_tree(tmp_path) + result = runner.invoke(app, ["stats", "*.py", "--base", str(tmp_path)]) + assert result.exit_code == 0 + assert "Files" in result.stdout + assert "Total size" in result.stdout + + +def test_stats_empty_tree(tmp_path): + """`stats` on an empty match still exits cleanly.""" + result = runner.invoke(app, ["stats", "*.py", "--base", str(tmp_path)]) + assert result.exit_code == 0 + + +def test_tree_subcommand(tmp_path): + """`rglob tree` renders a Unicode tree.""" + _build_tree(tmp_path) + result = runner.invoke(app, ["tree", "*.py", "--base", str(tmp_path)]) + assert result.exit_code == 0 + assert "a.py" in result.stdout + assert "nested.py" in result.stdout + + +def test_tree_shares_parent_dir(tmp_path): + """When multiple files share a subdir, the subdir node is reused.""" + sub = tmp_path / "sub" + sub.mkdir() + (sub / "a.py").write_text("") + (sub / "b.py").write_text("") + result = runner.invoke(app, ["tree", "*.py", "--base", str(tmp_path)]) + assert result.exit_code == 0 + # `sub/` should appear once even though two files are inside. + assert result.stdout.count("sub/") == 1 + + +def test_top_subcommand(tmp_path): + """`rglob top -n 2` returns the two largest files.""" + (tmp_path / "small.bin").write_bytes(b"x" * 10) + (tmp_path / "medium.bin").write_bytes(b"x" * 1000) + (tmp_path / "large.bin").write_bytes(b"x" * 100000) + result = runner.invoke(app, ["top", "*.bin", "--base", str(tmp_path), "-n", "2"]) + assert result.exit_code == 0 + assert "large.bin" in result.stdout + assert "medium.bin" in result.stdout + + +def test_dupes_subcommand_finds_duplicates(tmp_path): + """`rglob dupes` lists duplicate groups.""" + payload = b"identical content forever" + (tmp_path / "a.bin").write_bytes(payload) + (tmp_path / "b.bin").write_bytes(payload) + (tmp_path / "unique.bin").write_bytes(b"different") + result = runner.invoke(app, ["dupes", "*.bin", "--base", str(tmp_path)]) + assert result.exit_code == 0 + assert "a.bin" in result.stdout + assert "b.bin" in result.stdout + + +def test_find_type_filter(tmp_path): + """`rglob find -t f` matches only files.""" + (tmp_path / "f.txt").write_text("") + (tmp_path / "d").mkdir() + result = runner.invoke( + app, + ["find", "*", "--base", str(tmp_path), "-t", "f"], + ) + assert result.exit_code == 0 + assert "f.txt" in result.stdout + assert " d\n" not in result.stdout + + +def test_find_size_filter(tmp_path): + """`rglob find --min-size` excludes small files.""" + (tmp_path / "small.bin").write_bytes(b"x" * 10) + (tmp_path / "big.bin").write_bytes(b"x" * 5000) + result = runner.invoke( + app, + ["find", "*.bin", "--base", str(tmp_path), "--min-size", "1K"], + ) + assert result.exit_code == 0 + assert "big.bin" in result.stdout + assert "small.bin" not in result.stdout + + +def test_find_gitignore_flag(tmp_path): + """`rglob find --gitignore` honours .gitignore.""" + pytest.importorskip("pathspec") + (tmp_path / ".gitignore").write_text("*.log\n") + (tmp_path / "keep.py").write_text("") + (tmp_path / "skip.log").write_text("") + result = runner.invoke( + app, + ["find", "*", "--base", str(tmp_path), "--gitignore", "-H"], + ) + assert result.exit_code == 0 + assert "keep.py" in result.stdout + assert "skip.log" not in result.stdout + + +def test_dupes_subcommand_clean_tree(tmp_path): + """`rglob dupes` on a clean tree reports 'No duplicates found.'.""" + (tmp_path / "a.bin").write_bytes(b"unique-1") + (tmp_path / "b.bin").write_bytes(b"unique-2") + result = runner.invoke(app, ["dupes", "*.bin", "--base", str(tmp_path)]) + assert result.exit_code == 0 + assert "No duplicates" in result.stdout + + +def test_top_with_unreadable_file(tmp_path, monkeypatch): + """`top` skips files whose stat() raises OSError.""" + (tmp_path / "a.bin").write_bytes(b"x" * 1000) + (tmp_path / "b.bin").write_bytes(b"x" * 500) + real_stat = __import__("pathlib").Path.stat + + def boom(self, *a, **kw): + if self.name == "a.bin": + raise PermissionError("nope") + return real_stat(self, *a, **kw) + + monkeypatch.setattr("pathlib.Path.stat", boom) + result = runner.invoke(app, ["top", "*.bin", "--base", str(tmp_path)]) + assert result.exit_code == 0 + assert "b.bin" in result.stdout + + +def test_stats_with_unreadable_file(tmp_path, monkeypatch): + """`stats` aggregates a size of 0 for files whose stat() raises.""" + (tmp_path / "x.py").write_text("hi") + real_stat = __import__("pathlib").Path.stat + + def boom(self, *a, **kw): + if self.name == "x.py": + raise PermissionError("nope") + return real_stat(self, *a, **kw) + + monkeypatch.setattr("pathlib.Path.stat", boom) + result = runner.invoke(app, ["stats", "*.py", "--base", str(tmp_path)]) + assert result.exit_code == 0 diff --git a/tests/test_dupes.py b/tests/test_dupes.py new file mode 100644 index 0000000..8efe108 --- /dev/null +++ b/tests/test_dupes.py @@ -0,0 +1,103 @@ +"""Tests for the duplicate-detection module (Phase 5).""" + +from __future__ import annotations + +from rglob._dupes import find_duplicates + + +def test_no_duplicates_empty_tree(tmp_path): + """No files → no groups.""" + assert find_duplicates([]) == [] + + +def test_unique_files_no_groups(tmp_path): + """Two distinct files of the same size still aren't duplicates.""" + a = tmp_path / "a.bin" + b = tmp_path / "b.bin" + a.write_bytes(b"hello world!!!") # 14 bytes + b.write_bytes(b"goodbye world!") # 14 bytes + assert find_duplicates([a, b]) == [] + + +def test_exact_duplicates_grouped(tmp_path): + """Two identical files form one group.""" + a = tmp_path / "a.bin" + b = tmp_path / "b.bin" + payload = b"identical content" + a.write_bytes(payload) + b.write_bytes(payload) + groups = find_duplicates([a, b]) + assert len(groups) == 1 + assert set(groups[0]) == {a, b} + + +def test_three_way_duplicates(tmp_path): + """Three identical files form a single group of three.""" + payload = b"X" * 5000 # >4KB so head + full hash both run + files = [] + for name in ("a", "b", "c"): + p = tmp_path / f"{name}.bin" + p.write_bytes(payload) + files.append(p) + groups = find_duplicates(files) + assert len(groups) == 1 + assert set(groups[0]) == set(files) + + +def test_timeout_check_runs_during_hash_pipeline(tmp_path): + """Duplicate hashing invokes the optional timeout hook while reading.""" + payload = b"X" * 5000 + a = tmp_path / "a.bin" + b = tmp_path / "b.bin" + a.write_bytes(payload) + b.write_bytes(payload) + calls = 0 + + def check_timeout(): + nonlocal calls + calls += 1 + + groups = find_duplicates([a, b], timeout_check=check_timeout) + + assert len(groups) == 1 + assert set(groups[0]) == {a, b} + assert calls > 0 + + +def test_partial_overlap(tmp_path): + """First 4 KiB match but full bytes differ → not a group.""" + head = b"X" * 4096 + a = tmp_path / "a.bin" + b = tmp_path / "b.bin" + a.write_bytes(head + b"hello") + b.write_bytes(head + b"world") + assert find_duplicates([a, b]) == [] + + +def test_mixed_groups(tmp_path): + """Multiple disjoint groups are returned independently.""" + g1_payload = b"group-one-payload" + g2_payload = b"different-group-content" + a = tmp_path / "g1a.bin" + b = tmp_path / "g1b.bin" + c = tmp_path / "g2a.bin" + d = tmp_path / "g2b.bin" + e = tmp_path / "lonely.bin" + a.write_bytes(g1_payload) + b.write_bytes(g1_payload) + c.write_bytes(g2_payload) + d.write_bytes(g2_payload) + e.write_bytes(b"unique") + groups = find_duplicates([a, b, c, d, e]) + sets = [set(g) for g in groups] + assert {a, b} in sets + assert {c, d} in sets + # lonely.bin must not appear in any group + assert all(e not in s for s in sets) + + +def test_directory_inputs_skipped(tmp_path): + """Directory inputs are skipped (only files are hashed).""" + sub = tmp_path / "sub" + sub.mkdir() + assert find_duplicates([sub]) == [] diff --git a/tests/test_examples.py b/tests/test_examples.py new file mode 100644 index 0000000..aa47c35 --- /dev/null +++ b/tests/test_examples.py @@ -0,0 +1,195 @@ +"""Runnable harness for `docs/examples/*.md`. + +Walks every example file, extracts fenced code blocks that carry a +magic header line (`` or +``), and executes them. The +results land in pytest so `make test` fails if a published recipe rots. + +Untagged blocks are treated as documentation-only and are skipped. + +Recipes that need a sample tree reference the env var `RGLOB_FIXTURE`, +which the harness points at `tests/fixtures/agent-tree/` (the same +golden fixture the agent-contract tests use). This keeps recipes +portable across CI hosts without leaking developer paths into the +docs. + +Supported tag attributes (space-separated, all on one HTML comment): +- `name=` — required; used in the pytest test name. +- `runner=bash|python` — required; selects the execution backend. +- `expected_substring=` — optional; substring that must appear in + stdout. Quote with single quotes if it contains spaces. Multiple + `expected_substring=` attributes can appear. +- `expected_exit_code=` — optional; default 0. + +Example: + + + ```bash + rglob find "*.py" --base "$RGLOB_FIXTURE" --json + ``` +""" + +from __future__ import annotations + +import os +import re +import shlex +import subprocess +import sys +import textwrap +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +EXAMPLES_DIR = REPO_ROOT / "docs" / "examples" +FIXTURE_DIR = REPO_ROOT / "tests" / "fixtures" / "agent-tree" + +_TAG_RE = re.compile( + r"\s*\n```(?P\w+)?\s*\n(?P.*?)\n```", + re.DOTALL, +) +# Tokens are `key=value` pairs; value may be bare or single-quoted. +_ATTR_RE = re.compile(r"(?P\w+)=(?:'(?P[^']*)'|(?P\S+))") + + +@dataclass(frozen=True) +class Example: + """A single runnable example extracted from a Markdown file.""" + + source: Path + name: str + runner: str + body: str + expected_substrings: tuple[str, ...] + expected_exit_code: int + + @property + def test_id(self) -> str: + """Pytest parametrization id.""" + return f"{self.source.stem}::{self.name}" + + +def _parse_attrs(raw: str) -> dict[str, list[str]]: + """Parse the magic-comment attribute string into a dict of lists.""" + out: dict[str, list[str]] = {} + for match in _ATTR_RE.finditer(raw): + key = match.group("key") + value = match.group("qval") if match.group("qval") is not None else match.group("val") + out.setdefault(key, []).append(value) + return out + + +def _iter_examples(md_path: Path) -> Iterator[Example]: + """Yield every tagged example block from a Markdown file.""" + text = md_path.read_text(encoding="utf-8") + for match in _TAG_RE.finditer(text): + attrs = _parse_attrs(match.group("attrs")) + if "name" not in attrs or "runner" not in attrs: + continue + yield Example( + source=md_path, + name=attrs["name"][0], + runner=attrs["runner"][0], + body=match.group("body"), + expected_substrings=tuple(attrs.get("expected_substring", ())), + expected_exit_code=int(attrs.get("expected_exit_code", ["0"])[0]), + ) + + +def _discover() -> list[Example]: + """Walk `docs/examples/` and collect every tagged example.""" + if not EXAMPLES_DIR.exists(): + return [] + return [ex for md in sorted(EXAMPLES_DIR.glob("*.md")) for ex in _iter_examples(md)] + + +_EXAMPLES = _discover() + + +def _run_bash(example: Example) -> subprocess.CompletedProcess[str]: + """Run a bash-tagged example in a subprocess with `RGLOB_FIXTURE` set. + + Prepends the active Python's `bin/` (typically the project's `.venv`) + to PATH so the freshly-installed `rglob` entry point wins over any + stale user-wide shim. + """ + python_bin = Path(sys.executable).parent + env = { + **os.environ, + "PATH": f"{python_bin}{os.pathsep}{os.environ.get('PATH', '')}", + "RGLOB_FIXTURE": str(FIXTURE_DIR), + } + return subprocess.run( + ["bash", "-c", textwrap.dedent(example.body)], + capture_output=True, + text=True, + env=env, + cwd=REPO_ROOT, + check=False, + timeout=30, + ) + + +def _run_python(example: Example) -> subprocess.CompletedProcess[str]: + """Run a python-tagged example in a subprocess for isolation.""" + env = {**os.environ, "RGLOB_FIXTURE": str(FIXTURE_DIR)} + return subprocess.run( + [sys.executable, "-c", textwrap.dedent(example.body)], + capture_output=True, + text=True, + env=env, + cwd=REPO_ROOT, + check=False, + timeout=30, + ) + + +_RUNNERS = {"bash": _run_bash, "python": _run_python} + + +@pytest.mark.parametrize("example", _EXAMPLES, ids=lambda ex: ex.test_id) +def test_runnable_example(example: Example) -> None: + """Run a tagged example and validate stdout / exit code.""" + runner = _RUNNERS.get(example.runner) + if runner is None: + pytest.skip(f"unknown runner: {example.runner!r}") + if example.runner == "bash" and not _has_bash(): + pytest.skip("bash not available on this host") + + result = runner(example) + + assert result.returncode == example.expected_exit_code, ( + f"{example.test_id} exited with {result.returncode} " + f"(expected {example.expected_exit_code})\n" + f"--- stdout ---\n{result.stdout}\n--- stderr ---\n{result.stderr}" + ) + for needle in example.expected_substrings: + assert needle in result.stdout, ( + f"{example.test_id} stdout missing {needle!r}\n--- stdout ---\n{result.stdout}" + ) + + +def _has_bash() -> bool: + """Return whether a working `bash` is on PATH. + + Windows ships a `bash.exe` shim in System32 that forwards to WSL; on + CI runners without a WSL distro installed that shim exists but exits + non-zero. Require a zero exit from `bash -c true` so the stub falls + through and the bash-tagged examples are skipped instead of failing. + """ + try: + result = subprocess.run( + ["bash", "-c", "true"], + capture_output=True, + check=False, + timeout=5, + ) + except (FileNotFoundError, subprocess.TimeoutExpired, OSError): + return False + return result.returncode == 0 + + +_ = shlex # kept for future quoting helpers; silence unused-import lint diff --git a/tests/test_filters.py b/tests/test_filters.py new file mode 100644 index 0000000..92fb8aa --- /dev/null +++ b/tests/test_filters.py @@ -0,0 +1,366 @@ +"""Tests for the Phase 5 filter additions: kinds, size, mtime, gitignore.""" + +from __future__ import annotations + +import os +import time +from datetime import UTC, datetime, timedelta + +import pytest + +from rglob import find_all +from rglob._filters import ( + kinds_match, + mtime_predicate, + owner_predicate, + parse_perm, + parse_size, + parse_time, + perm_predicate, + size_predicate, +) + +posix_only = pytest.mark.skipif( + os.name != "posix", + reason="Test relies on POSIX mode bits not honoured by NTFS", +) + +# ─── parse_size ─────────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (1024, 1024), + ("1024", 1024), + ("1K", 1024), + ("1KB", 1024), + ("1KiB", 1024), + ("2.5M", int(2.5 * 2**20)), + ("1G", 2**30), + ("100 kb", 100 * 1024), + (1024.5, 1024), + ], +) +def test_parse_size(value, expected): + assert parse_size(value) == expected + + +def test_parse_size_invalid(): + with pytest.raises(ValueError, match="unparseable size"): + parse_size("nope") + + +# ─── parse_perm ────────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("644", (0o644, "exact")), + ("0o755", (0o755, "exact")), + ("-111", (0o111, "all")), + ("/222", (0o222, "any")), + (0o600, (0o600, "exact")), + ], +) +def test_parse_perm(value, expected): + assert parse_perm(value) == expected + + +def test_parse_perm_invalid(): + with pytest.raises(ValueError, match="unparseable permission mode"): + parse_perm("nope") + + +# ─── parse_time ─────────────────────────────────────────────────────────────── + + +def test_parse_time_datetime_passthrough(): + now = datetime.now(UTC) + assert parse_time(now) == now + + +def test_parse_time_naive_datetime_gets_utc(): + naive = datetime(2024, 1, 1) + parsed = parse_time(naive) + assert parsed.tzinfo is not None + + +def test_parse_time_timedelta_is_relative(): + parsed = parse_time(timedelta(hours=1)) + now = datetime.now(UTC) + assert (now - parsed) >= timedelta(minutes=55) + + +def test_parse_time_duration_string(): + parsed = parse_time("7d") + now = datetime.now(UTC) + assert (now - parsed) >= timedelta(days=6, hours=23) + + +def test_parse_time_iso_date(): + parsed = parse_time("2024-01-15") + assert parsed.year == 2024 + assert parsed.month == 1 + assert parsed.day == 15 + + +def test_parse_time_invalid(): + with pytest.raises(ValueError, match="unparseable time"): + parse_time("not-a-time") + + +# ─── kinds ─────────────────────────────────────────────────────────────────── + + +def test_kinds_file_only(tmp_path): + """`kinds={'f'}` returns only regular files.""" + (tmp_path / "a.txt").write_text("") + (tmp_path / "sub").mkdir() + (tmp_path / "sub" / "b.txt").write_text("") + out = find_all(tmp_path, "*", kinds={"f"}) + assert all(p.is_file() for p in out) + assert {p.name for p in out} == {"a.txt", "b.txt"} + + +def test_kinds_dir_only(tmp_path): + """`kinds={'d'}` returns only directories.""" + (tmp_path / "a.txt").write_text("") + (tmp_path / "sub").mkdir() + out = find_all(tmp_path, "*", kinds={"d"}) + assert {p.name for p in out} == {"sub"} + + +@posix_only +def test_kinds_executable(tmp_path): + """`kinds={'x'}` matches files with the exec bit set.""" + pytest.importorskip("os") # always available; just to make linter happy + script = tmp_path / "run.sh" + script.write_text("#!/bin/sh\n") + script.chmod(0o755) + plain = tmp_path / "data.txt" + plain.write_text("") + out = find_all(tmp_path, "*", kinds={"x"}) + names = {p.name for p in out} + assert "run.sh" in names + assert "data.txt" not in names + + +def test_kinds_symlink(tmp_path): + """`kinds={'l'}` matches symbolic links.""" + real = tmp_path / "real.txt" + real.write_text("hi") + try: + (tmp_path / "link.txt").symlink_to(real) + except (OSError, NotImplementedError): # pragma: no cover - Windows-only + pytest.skip("symlinks not permitted") + out = find_all(tmp_path, "*", kinds={"l"}) + assert {p.name for p in out} == {"link.txt"} + + +def test_default_entry_predicates_pass(tmp_path): + """Inactive entry predicates remain explicit pass-through helpers.""" + target = tmp_path / "a.txt" + target.write_text("") + with os.scandir(tmp_path) as entries: + entry = next(item for item in entries if item.name == target.name) + + assert kinds_match(entry, ()) + assert kinds_match(entry, ("f",)) # exercises _entry_kind() via legacy path + assert size_predicate(None, None)(entry) + assert mtime_predicate(None, None)(entry) + assert perm_predicate(None)(entry) + assert owner_predicate(None, None)(entry) + + +def test_kinds_predicate_empty_set_is_match_all(): + """`kinds_predicate(())` returns a match-everything closure.""" + from rglob._filters import kinds_predicate + + pred = kinds_predicate(()) + # The closure must accept any input and return truthy; it's only + # called by the walker when no other filter has fired. + assert pred(object()) is True # type: ignore[arg-type] + + +def test_kinds_predicate_multi_kind_uses_union(tmp_path): + """A multi-kind set falls back to `_entry_kind` union semantics. + + Includes a symlink so the `tags.add("l")` branch in `_entry_kind` + is exercised — single-kind paths bypass that helper entirely now. + """ + (tmp_path / "file.txt").write_text("") + (tmp_path / "sub").mkdir() + target = tmp_path / "target.txt" + target.write_text("") + try: + (tmp_path / "link.txt").symlink_to(target) + except (OSError, NotImplementedError): # pragma: no cover - Windows-only + pytest.skip("symlinks not permitted on this host") + out = find_all(tmp_path, "*", kinds={"f", "d", "l"}) + names = {p.name for p in out} + assert names == {"file.txt", "sub", "target.txt", "link.txt"} + + +# ─── size filters ──────────────────────────────────────────────────────────── + + +def test_min_size_filters_small_files(tmp_path): + """`min_size='1K'` excludes files under 1 KiB.""" + (tmp_path / "tiny.bin").write_bytes(b"x" * 100) + (tmp_path / "big.bin").write_bytes(b"x" * 2048) + out = find_all(tmp_path, "*.bin", min_size="1K") + assert {p.name for p in out} == {"big.bin"} + + +def test_max_size_filters_large_files(tmp_path): + """`max_size=500` excludes files over 500 bytes.""" + (tmp_path / "tiny.bin").write_bytes(b"x" * 100) + (tmp_path / "big.bin").write_bytes(b"x" * 2048) + out = find_all(tmp_path, "*.bin", max_size=500) + assert {p.name for p in out} == {"tiny.bin"} + + +def test_size_filter_range(tmp_path): + """Combining `min_size` and `max_size` works as a closed range.""" + (tmp_path / "small.bin").write_bytes(b"x" * 100) + (tmp_path / "mid.bin").write_bytes(b"x" * 1000) + (tmp_path / "big.bin").write_bytes(b"x" * 5000) + out = find_all(tmp_path, "*.bin", min_size=500, max_size=2000) + assert {p.name for p in out} == {"mid.bin"} + + +# ─── mtime filters ─────────────────────────────────────────────────────────── + + +def test_newer_than_filter(tmp_path): + """`newer_than` only yields files modified after the cutoff.""" + old = tmp_path / "old.txt" + old.write_text("") + old_mtime = time.time() - 86400 * 7 # 7 days ago + os.utime(old, (old_mtime, old_mtime)) + new = tmp_path / "new.txt" + new.write_text("") + out = find_all(tmp_path, "*.txt", newer_than="1d") + assert {p.name for p in out} == {"new.txt"} + + +def test_older_than_filter(tmp_path): + """`older_than` only yields files modified before the cutoff.""" + old = tmp_path / "old.txt" + old.write_text("") + old_mtime = time.time() - 86400 * 7 + os.utime(old, (old_mtime, old_mtime)) + new = tmp_path / "new.txt" + new.write_text("") + out = find_all(tmp_path, "*.txt", older_than="1d") + assert {p.name for p in out} == {"old.txt"} + + +def test_newer_than_file_filter(tmp_path): + """`newer_than_file` compares mtimes against a reference file.""" + reference = tmp_path / "reference.ref" + reference.write_text("") + ref_mtime = time.time() - 86400 + os.utime(reference, (ref_mtime, ref_mtime)) + old = tmp_path / "old.txt" + old.write_text("") + old_mtime = time.time() - 86400 * 2 + os.utime(old, (old_mtime, old_mtime)) + new = tmp_path / "new.txt" + new.write_text("") + + out = find_all(tmp_path, "*.txt", newer_than_file=reference) + assert {p.name for p in out} == {"new.txt"} + + +# ─── find(1)-style permission / owner filters ──────────────────────────────── + + +@posix_only +def test_perm_exact_filter(tmp_path): + """`perm='600'` matches exact POSIX mode bits.""" + private = tmp_path / "private.txt" + private.write_text("") + private.chmod(0o600) + public = tmp_path / "public.txt" + public.write_text("") + public.chmod(0o644) + + out = find_all(tmp_path, "*.txt", perm="600") + assert {p.name for p in out} == {"private.txt"} + + +@posix_only +def test_perm_all_and_any_filters(tmp_path): + """Leading '-' and '/' implement all-bits and any-bits matching.""" + executable = tmp_path / "run.sh" + executable.write_text("") + executable.chmod(0o755) + plain = tmp_path / "plain.sh" + plain.write_text("") + plain.chmod(0o644) + + assert {p.name for p in find_all(tmp_path, "*.sh", perm="-111")} == {"run.sh"} + assert {p.name for p in find_all(tmp_path, "*.sh", perm="/111")} == {"run.sh"} + + +def test_uid_gid_filters(tmp_path): + """POSIX uid/gid filters match the current process owner.""" + if os.name != "posix": + pytest.skip("uid/gid filters are POSIX-only") + target = tmp_path / "owned.txt" + target.write_text("") + out = find_all(tmp_path, "*.txt", uid=os.getuid(), gid=os.getgid()) + assert {p.name for p in out} == {"owned.txt"} + + +def test_uid_filter_rejects_non_matching_owner(tmp_path): + """A non-matching uid excludes otherwise matching files.""" + if os.name != "posix": + pytest.skip("uid/gid filters are POSIX-only") + target = tmp_path / "owned.txt" + target.write_text("") + out = find_all(tmp_path, "*.txt", uid=os.getuid() + 1) + assert out == [] + + +# ─── .gitignore awareness ──────────────────────────────────────────────────── + + +def test_respect_gitignore(tmp_path): + """`respect_gitignore=True` honours .gitignore patterns.""" + pytest.importorskip("pathspec") + (tmp_path / ".gitignore").write_text("*.log\nbuild/\n") + (tmp_path / "keep.py").write_text("") + (tmp_path / "skip.log").write_text("") + (tmp_path / "build").mkdir() + (tmp_path / "build" / "artifact.bin").write_text("") + out = find_all(tmp_path, "*", respect_gitignore=True, hidden=True) + names = {p.name for p in out} + assert "keep.py" in names + assert "skip.log" not in names + # build/ itself or its contents should be ignored + assert "artifact.bin" not in names + + +def test_size_filter_passes_directories(tmp_path): + """Size filters always pass directories (they have no inherent file size).""" + (tmp_path / "sub").mkdir() + (tmp_path / "sub" / "child.bin").write_bytes(b"x" * 5000) + out = find_all(tmp_path, "*", min_size="1K") + names = {p.name for p in out} + # The dir entry should still be in the results despite having no "size". + assert "sub" in names + assert "child.bin" in names + + +def test_no_gitignore_by_default(tmp_path): + """Without `respect_gitignore`, gitignored files are still listed.""" + (tmp_path / ".gitignore").write_text("*.log\n") + (tmp_path / "keep.py").write_text("") + (tmp_path / "skip.log").write_text("") + out = find_all(tmp_path, "*", hidden=True) + names = {p.name for p in out} + assert "skip.log" in names diff --git a/tests/test_find.py b/tests/test_find.py new file mode 100644 index 0000000..504e546 --- /dev/null +++ b/tests/test_find.py @@ -0,0 +1,414 @@ +"""Tests for the new `find()` / `find_all()` modern API (Phase 3).""" + +from __future__ import annotations + +import os +import sys +import warnings +from pathlib import Path + +import pytest + +from rglob import find, find_all + +# ─── Basic matching ────────────────────────────────────────────────────────── + + +def test_find_returns_iterator(tmp_path): + """`find` is a lazy iterator, not a list.""" + (tmp_path / "a.txt").write_text("a") + result = find(tmp_path, "*.txt") + assert iter(result) is result or hasattr(result, "__next__") + + +def test_find_all_returns_list_of_paths(tmp_path): + """`find_all` materialises into `list[Path]`.""" + (tmp_path / "a.txt").write_text("a") + (tmp_path / "b.txt").write_text("b") + out = find_all(tmp_path, "*.txt") + assert isinstance(out, list) + assert all(isinstance(p, Path) for p in out) + assert {p.name for p in out} == {"a.txt", "b.txt"} + + +def test_find_basename_pattern_matches_recursively(tmp_path): + """Basename patterns recurse: `*.py` matches at any depth.""" + (tmp_path / "top.py").write_text("") + sub = tmp_path / "sub" + sub.mkdir() + (sub / "nested.py").write_text("") + assert {p.name for p in find_all(tmp_path, "*.py")} == {"top.py", "nested.py"} + + +def test_find_double_star_matches_any_depth(tmp_path): + """`**/*.py` matches Python files at any depth.""" + (tmp_path / "top.py").write_text("") + (tmp_path / "sub").mkdir() + (tmp_path / "sub" / "nested.py").write_text("") + (tmp_path / "sub" / "deep").mkdir() + (tmp_path / "sub" / "deep" / "deeper.py").write_text("") + assert {p.name for p in find_all(tmp_path, "**/*.py")} == { + "top.py", + "nested.py", + "deeper.py", + } + + +def test_find_path_pattern_matches_subdir_prefix(tmp_path): + """Patterns with `/` match against the relative path.""" + (tmp_path / "a").mkdir() + (tmp_path / "a" / "x.txt").write_text("") + (tmp_path / "b").mkdir() + (tmp_path / "b" / "x.txt").write_text("") + out = find_all(tmp_path, "a/*.txt") + assert {p.name for p in out} == {"x.txt"} + assert all("a" in p.parts for p in out) + + +def test_find_multiple_patterns(tmp_path): + """Sequence of patterns acts as OR.""" + (tmp_path / "a.py").write_text("") + (tmp_path / "b.txt").write_text("") + (tmp_path / "c.md").write_text("") + out = find_all(tmp_path, ["*.py", "*.txt"]) + assert {p.name for p in out} == {"a.py", "b.txt"} + + +def test_find_default_pattern_matches_everything(tmp_path): + """No pattern → match all entries.""" + (tmp_path / "a").mkdir() + (tmp_path / "a" / "x.txt").write_text("") + (tmp_path / "b.py").write_text("") + out = {p.name for p in find_all(tmp_path)} + assert out == {"a", "x.txt", "b.py"} + + +# ─── Exclude ───────────────────────────────────────────────────────────────── + + +def test_exclude_prunes_directory(tmp_path): + """Excluded directories are not descended into.""" + (tmp_path / "keep").mkdir() + (tmp_path / "keep" / "x.py").write_text("") + (tmp_path / "skip").mkdir() + (tmp_path / "skip" / "y.py").write_text("") + out = find_all(tmp_path, "*.py", exclude="skip") + assert {p.name for p in out} == {"x.py"} + + +def test_exclude_multiple_patterns(tmp_path): + """Multiple exclude patterns are OR'd.""" + (tmp_path / "a.tmp").write_text("") + (tmp_path / "b.bak").write_text("") + (tmp_path / "c.py").write_text("") + out = find_all(tmp_path, "*", exclude=["*.tmp", "*.bak"]) + assert {p.name for p in out} == {"c.py"} + + +# ─── max_depth ─────────────────────────────────────────────────────────────── + + +def test_max_depth_zero(tmp_path): + """`max_depth=0` limits to direct children of base.""" + (tmp_path / "top.txt").write_text("") + (tmp_path / "sub").mkdir() + (tmp_path / "sub" / "nested.txt").write_text("") + out = find_all(tmp_path, "*.txt", max_depth=0) + assert {p.name for p in out} == {"top.txt"} + + +def test_max_depth_one(tmp_path): + """`max_depth=1` includes one level of subdir.""" + (tmp_path / "top.txt").write_text("") + (tmp_path / "sub").mkdir() + (tmp_path / "sub" / "nested.txt").write_text("") + (tmp_path / "sub" / "deep").mkdir() + (tmp_path / "sub" / "deep" / "deeper.txt").write_text("") + out = find_all(tmp_path, "*.txt", max_depth=1) + assert {p.name for p in out} == {"top.txt", "nested.txt"} + + +def test_max_depth_none_unbounded(tmp_path): + """`max_depth=None` (default) walks arbitrarily deep.""" + chain = tmp_path + for i in range(5): + chain = chain / f"d{i}" + chain.mkdir() + (chain / "deep.txt").write_text("") + assert {p.name for p in find_all(tmp_path, "*.txt")} == {"deep.txt"} + + +# ─── hidden ────────────────────────────────────────────────────────────────── + + +def test_hidden_skipped_by_default(tmp_path): + """Dotfiles are skipped unless `hidden=True`.""" + (tmp_path / "visible.py").write_text("") + (tmp_path / ".hidden.py").write_text("") + (tmp_path / ".dotdir").mkdir() + (tmp_path / ".dotdir" / "x.py").write_text("") + out = {p.name for p in find_all(tmp_path, "*.py")} + assert out == {"visible.py"} + + +def test_hidden_included_when_enabled(tmp_path): + """`hidden=True` reveals dotfiles and dot-directory contents.""" + (tmp_path / "visible.py").write_text("") + (tmp_path / ".hidden.py").write_text("") + (tmp_path / ".dotdir").mkdir() + (tmp_path / ".dotdir" / "x.py").write_text("") + out = {p.name for p in find_all(tmp_path, "*.py", hidden=True)} + assert out == {"visible.py", ".hidden.py", "x.py"} + + +# ─── case_sensitive ────────────────────────────────────────────────────────── + + +def test_case_sensitive_true(tmp_path): + """Case-sensitive `True` rejects mismatched casing.""" + (tmp_path / "README.MD").write_text("") + assert find_all(tmp_path, "*.md", case_sensitive=True) == [] + assert len(find_all(tmp_path, "*.MD", case_sensitive=True)) == 1 + + +def test_case_sensitive_false(tmp_path, case_sensitive_fs): + """Case-sensitive `False` accepts either casing.""" + if not case_sensitive_fs: + # APFS/NTFS collapse README.MD and readme.md to one inode; the + # case-insensitive matching contract is exercised by + # test_case_sensitive_none_follows_os on those hosts. + pytest.skip("filesystem is case-insensitive; cannot create both casings") + (tmp_path / "README.MD").write_text("") + (tmp_path / "readme.md").write_text("") + out = find_all(tmp_path, "*.md", case_sensitive=False) + assert {p.name for p in out} == {"README.MD", "readme.md"} + + +def test_case_sensitive_none_follows_os(tmp_path): + """Default `None` matches the host OS convention.""" + (tmp_path / "README.MD").write_text("") + if sys.platform in ("win32", "darwin"): + # Case-insensitive default: lowercase pattern still matches. + assert len(find_all(tmp_path, "*.md")) == 1 + else: + # Linux: case-sensitive default. + assert find_all(tmp_path, "*.md") == [] + + +# ─── sort ──────────────────────────────────────────────────────────────────── + + +def test_sort_default_lexical(tmp_path): + """Default order is lexically sorted within each directory.""" + for name in ("c", "a", "b"): + (tmp_path / f"{name}.txt").write_text("") + out = [p.name for p in find_all(tmp_path, "*.txt")] + assert out == ["a.txt", "b.txt", "c.txt"] + + +def test_sort_false_raw_order(tmp_path): + """`sort=False` returns scandir's native order (we just don't assert ordering).""" + for name in ("c", "a", "b"): + (tmp_path / f"{name}.txt").write_text("") + out = find_all(tmp_path, "*.txt", sort=False) + assert {p.name for p in out} == {"a.txt", "b.txt", "c.txt"} + + +# ─── performance-sensitive fast paths ──────────────────────────────────────── + + +def test_find_avoids_pathlib_relative_to_per_entry(tmp_path, monkeypatch): + """The walker carries relative strings instead of recomputing Path.relative_to.""" + (tmp_path / "pkg").mkdir() + (tmp_path / "pkg" / "mod.py").write_text("") + + def boom(self, *other): + raise AssertionError(f"relative_to should not be called for {self!s} / {other!r}") + + monkeypatch.setattr(Path, "relative_to", boom) + + out = find_all(tmp_path, "**/*.py") + + assert [path.name for path in out] == ["mod.py"] + + +def test_find_avoids_realpath_when_not_following_symlinks(tmp_path, monkeypatch): + """Cycle detection realpath work is only needed when following symlinks.""" + (tmp_path / "pkg").mkdir() + (tmp_path / "pkg" / "mod.py").write_text("") + + def boom(_path): + raise AssertionError("realpath should not be called unless follow_symlinks=True") + + monkeypatch.setattr(os.path, "realpath", boom) + + out = find_all(tmp_path, "*.py") + + assert [path.name for path in out] == ["mod.py"] + + +# ─── on_error ──────────────────────────────────────────────────────────────── + + +def test_on_error_raise_propagates(tmp_path, monkeypatch): + """`on_error='raise'` re-raises OSError from scandir.""" + + def boom(_path): + raise PermissionError("nope") + + monkeypatch.setattr(os, "scandir", boom) + with pytest.raises(PermissionError): + list(find(tmp_path, "*", on_error="raise")) + + +def test_on_error_warn_emits_warning(tmp_path, monkeypatch): + """`on_error='warn'` (default) emits a RuntimeWarning.""" + + def boom(_path): + raise PermissionError("nope") + + monkeypatch.setattr(os, "scandir", boom) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + out = list(find(tmp_path, "*", on_error="warn")) + assert out == [] + assert any(issubclass(w.category, RuntimeWarning) for w in caught) + + +def test_on_error_ignore_silences(tmp_path, monkeypatch): + """`on_error='ignore'` swallows OSError entirely.""" + + def boom(_path): + raise PermissionError("nope") + + monkeypatch.setattr(os, "scandir", boom) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + out = list(find(tmp_path, "*", on_error="ignore")) + assert out == [] + assert not caught + + +# ─── follow_symlinks + loop detection ──────────────────────────────────────── + + +def test_symlinks_not_followed_by_default(tmp_path): + """Default `follow_symlinks=False` skips symlinked directories.""" + target = tmp_path / "real" + target.mkdir() + (target / "x.py").write_text("") + try: + (tmp_path / "link").symlink_to(target, target_is_directory=True) + except (OSError, NotImplementedError): # pragma: no cover - Windows-only + pytest.skip("symlinks not permitted") + out = {p.name for p in find_all(tmp_path, "*.py")} + assert out == {"x.py"} # Only via the real path, not the link. + + +def test_symlinks_followed_when_enabled(tmp_path): + """`follow_symlinks=True` traverses symlinked directories.""" + target = tmp_path / "real" + target.mkdir() + (target / "x.py").write_text("") + try: + (tmp_path / "link").symlink_to(target, target_is_directory=True) + except (OSError, NotImplementedError): # pragma: no cover - Windows-only + pytest.skip("symlinks not permitted") + out = find_all(tmp_path, "*.py", follow_symlinks=True) + # x.py via /real/x.py — the realpath memo prevents double-yield via /link/x.py. + assert len(out) == 1 + assert out[0].name == "x.py" + + +def test_symlink_loop_terminates(make_symlink_loop): + """A `a → b → a` cycle does not infinitely recurse.""" + entry = make_symlink_loop + # Should terminate quickly without exhausting recursion limits. + out = find_all(entry, "*", follow_symlinks=True) + # We don't assert exact contents — just that the walk terminated. + assert isinstance(out, list) + + +# ─── Path / PathLike input ─────────────────────────────────────────────────── + + +def test_base_accepts_str_and_pathlike(tmp_path): + """`base` accepts both str and Path.""" + (tmp_path / "a.txt").write_text("") + via_str = find_all(str(tmp_path), "*.txt") + via_path = find_all(tmp_path, "*.txt") + assert via_str == via_path + + +# ─── Pattern edge cases ────────────────────────────────────────────────────── + + +def test_slash_doublestar_at_end(tmp_path): + """`dir/**` matches anything under `dir/`.""" + (tmp_path / "keep").mkdir() + (tmp_path / "keep" / "x.py").write_text("") + (tmp_path / "keep" / "deep").mkdir() + (tmp_path / "keep" / "deep" / "y.py").write_text("") + (tmp_path / "other.py").write_text("") + out = {p.name for p in find_all(tmp_path, "keep/**")} + # Should include everything under keep/, plus the keep dir itself if + # it matches; behaviour we want is "everything *inside* keep". + assert "x.py" in out + assert "y.py" in out + assert "other.py" not in out + + +def test_bare_doublestar(tmp_path): + """Bare `**` matches every path.""" + (tmp_path / "a").mkdir() + (tmp_path / "a" / "x.py").write_text("") + (tmp_path / "b.txt").write_text("") + out = {p.name for p in find_all(tmp_path, "**")} + assert out == {"a", "x.py", "b.txt"} + + +def test_is_dir_error_handled(tmp_path, monkeypatch): + """When `DirEntry.is_dir()` raises, on_error is called and we move on.""" + (tmp_path / "a").mkdir() + (tmp_path / "a" / "x.py").write_text("") + (tmp_path / "b.py").write_text("") + + real_scandir = os.scandir + + class BrokenEntry: + """Wrap a real DirEntry but make is_dir raise.""" + + def __init__(self, inner): + self._inner = inner + self.name = inner.name + self.path = inner.path + + def is_dir(self, follow_symlinks=True): + if self.name == "a": + raise PermissionError("simulated") + return self._inner.is_dir(follow_symlinks=follow_symlinks) + + def is_symlink(self): + return self._inner.is_symlink() + + class FakeScandirCM: + def __init__(self, real_cm): + self._real_cm = real_cm + self._real_it = None + + def __enter__(self): + self._real_it = self._real_cm.__enter__() + return (BrokenEntry(e) for e in self._real_it) + + def __exit__(self, *args): + return self._real_cm.__exit__(*args) + + def fake_scandir(path): + return FakeScandirCM(real_scandir(path)) + + monkeypatch.setattr(os, "scandir", fake_scandir) + # With on_error='ignore' the bad entry is silently skipped. + out = {p.name for p in find_all(tmp_path, "*.py", on_error="ignore")} + # `a/x.py` should be missing (we couldn't recurse), `b.py` should be present. + assert "b.py" in out + assert "x.py" not in out diff --git a/tests/test_grep.py b/tests/test_grep.py new file mode 100644 index 0000000..5163568 --- /dev/null +++ b/tests/test_grep.py @@ -0,0 +1,715 @@ +"""Tests for the structured grep and count helpers.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from rglob._count import count_all +from rglob._grep import grep_all +from rglob.agent._models import CountOptions, GrepOptions + + +def test_grep_all_regex_with_context(tmp_path): + """Regex grep returns line matches with before/after context.""" + path = tmp_path / "notes.txt" + path.write_text("before\nTODO: one\nafter\n", encoding="utf-8") + + result = grep_all(GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path, context=1)) + + assert result.truncated is False + assert result.total_files_searched == 1 + assert result.results[0].line_number == 2 + assert result.results[0].before == ["before"] + assert result.results[0].after == ["after"] + + +def test_grep_all_after_context_spans_multiple_lines(tmp_path): + """`--after 2` keeps the match pending until two trailing lines arrive. + + Exercises the streaming branch where `_PendingMatch.feed` returns + False (still needs more lines) and the match goes back to the + pending list rather than into results. + """ + path = tmp_path / "notes.txt" + path.write_text("alpha\nTODO: one\nfollow1\nfollow2\nfollow3\n", encoding="utf-8") + + result = grep_all(GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path, after=2)) + + assert result.results[0].after == ["follow1", "follow2"] + + +def test_grep_all_after_context_invert(tmp_path): + """`--invert` works on the after-context branch too.""" + path = tmp_path / "notes.txt" + path.write_text("keep\nTODO\nkeep2\n", encoding="utf-8") + + result = grep_all( + GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path, after=1, invert=True) + ) + + # invert=True means non-TODO lines match. With after=1 each match + # captures the next line. + contents = [m.content for m in result.results] + assert "keep" in contents + + +def test_grep_all_after_context_limit_terminates(tmp_path): + """`--limit` short-circuits the after-context loop when the budget is met.""" + path = tmp_path / "notes.txt" + path.write_text("TODO one\nfollow1\nTODO two\nfollow2\n", encoding="utf-8") + + result = grep_all(GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path, after=1, limit=1)) + + assert result.truncated is True + assert result.truncated_reason == "limit" + assert len(result.results) == 1 + + +def test_grep_all_after_context_max_count_terminates(tmp_path): + """`--max-count` works on the after-context branch.""" + path = tmp_path / "notes.txt" + path.write_text("TODO one\nfollow1\nTODO two\nfollow2\n", encoding="utf-8") + + result = grep_all( + GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path, after=1, max_count=1) + ) + + assert result.truncated is True + assert result.truncated_reason == "max_count" + assert len(result.results) == 1 + + +def test_grep_all_after_context_line_timeout(monkeypatch, tmp_path): + """A line-level timeout is reported correctly in the after-context branch.""" + path = tmp_path / "notes.txt" + path.write_text("TODO\nfollow\n", encoding="utf-8") + + from rglob import _grep + + call_count = {"n": 0} + real_check = _grep.check_timeout + + def maybe_timeout(deadline) -> None: + call_count["n"] += 1 + if call_count["n"] >= 2: # let the file-open check pass; trip at first line + raise TimeoutError + real_check(deadline) + + monkeypatch.setattr(_grep, "check_timeout", maybe_timeout) + result = grep_all(GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path, after=1)) + + assert result.truncated is True + assert result.truncated_reason == "timeout" + assert any(err.code == "TIMEOUT" for err in result.errors) + + +def test_grep_all_after_context_line_timeout_silent(monkeypatch, tmp_path): + """`include_errors=False` suppresses the line-level timeout error envelope.""" + path = tmp_path / "notes.txt" + path.write_text("TODO\nfollow\n", encoding="utf-8") + + from rglob import _grep + + call_count = {"n": 0} + real_check = _grep.check_timeout + + def maybe_timeout(deadline) -> None: + call_count["n"] += 1 + if call_count["n"] >= 2: + raise TimeoutError + real_check(deadline) + + monkeypatch.setattr(_grep, "check_timeout", maybe_timeout) + result = grep_all( + GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path, after=1, include_errors=False) + ) + + assert result.truncated is True + assert result.truncated_reason == "timeout" + assert result.errors == [] + + +def test_grep_files_with_matches_returns_one_per_file(tmp_path): + """`files_with_matches=True` emits one stub LineMatch per matching file.""" + (tmp_path / "a.txt").write_text("TODO one\nTODO two\nfine\n") + (tmp_path / "b.txt").write_text("nope\n") + (tmp_path / "c.txt").write_text("TODO three\n") + + result = grep_all( + GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path, files_with_matches=True) + ) + + paths = sorted(m.path.name for m in result.results) + assert paths == ["a.txt", "c.txt"] + # Stubs carry no content / line_number / context — the agent learns + # only that the file matched. + for stub in result.results: + assert stub.line_number == 0 + assert stub.content == "" + assert stub.before == [] + assert stub.after == [] + + +def test_grep_files_with_matches_respects_limit(tmp_path): + """`--limit` bounds the number of matching FILES emitted.""" + for i in range(5): + (tmp_path / f"f{i}.txt").write_text("TODO\n") + + result = grep_all( + GrepOptions( + pattern="TODO", paths=["*.txt"], base=tmp_path, files_with_matches=True, limit=2 + ) + ) + + assert len(result.results) == 2 + assert result.truncated is True + assert result.truncated_reason == "limit" + + +def test_grep_count_only_emits_per_file_counts(tmp_path): + """`count_only=True` emits one stub per file with `line_number` = match count.""" + (tmp_path / "a.txt").write_text("TODO\nTODO\nfine\nTODO\n") + (tmp_path / "b.txt").write_text("nothing here\n") + (tmp_path / "c.txt").write_text("TODO\n") + + result = grep_all(GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path, count_only=True)) + + counts = {m.path.name: m.line_number for m in result.results} + assert counts == {"a.txt": 3, "c.txt": 1} + + +def test_grep_count_only_respects_limit(tmp_path): + """`--limit` bounds the number of FILES with non-zero counts.""" + for i in range(5): + (tmp_path / f"f{i}.txt").write_text("TODO\nTODO\n") + + result = grep_all( + GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path, count_only=True, limit=2) + ) + + assert len(result.results) == 2 + assert result.truncated is True + assert result.truncated_reason == "limit" + + +def test_grep_iter_yields_matches(tmp_path): + """`grep_iter` yields the same LineMatch records that grep_all returns.""" + from rglob._grep import grep_iter + + (tmp_path / "a.txt").write_text("alpha\nTODO\nbeta\n") + matches = list(grep_iter(GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path))) + assert len(matches) == 1 + assert matches[0].content == "TODO" + + +def test_grep_all_streams_large_text_file(tmp_path): + """Files larger than the streaming threshold use TextIOWrapper iteration.""" + big = tmp_path / "big.txt" + # > 64 KiB so we cross _STREAM_THRESHOLD_BYTES; sprinkle a TODO at the top. + big.write_text("TODO at top\n" + ("filler\n" * 12_000), encoding="utf-8") + + result = grep_all(GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path)) + + assert len(result.results) == 1 + assert result.results[0].line_number == 1 + assert result.bytes_read >= 64 * 1024 + + +def test_grep_all_streams_large_binary_file_is_skipped(tmp_path): + """A large binary file is detected by the head probe and skipped.""" + big = tmp_path / "big.bin" + # 100 KB, with NULs in the first KB so binary detection short-circuits + # before we instantiate TextIOWrapper at all. + big.write_bytes(b"\x00\x00TODO\x00\x00" + b"x" * 100_000) + + result = grep_all(GrepOptions(pattern="TODO", paths=["*.bin"], base=tmp_path)) + + assert result.results == [] + assert result.errors[0].code == "BINARY" + + +def test_grep_all_streams_large_binary_silent(tmp_path): + """`include_errors=False` suppresses the BINARY envelope on the streaming path.""" + big = tmp_path / "big.bin" + big.write_bytes(b"\x00\x00TODO\x00\x00" + b"x" * 100_000) + + result = grep_all( + GrepOptions(pattern="TODO", paths=["*.bin"], base=tmp_path, include_errors=False) + ) + + assert result.results == [] + assert result.errors == [] + + +def test_grep_all_outer_timeout_fires_before_first_file(monkeypatch, tmp_path): + """A timeout at the outer per-file check produces a global TIMEOUT envelope.""" + (tmp_path / "a.txt").write_text("TODO\n", encoding="utf-8") + from rglob import _grep + + def always_timeout(_deadline) -> None: + raise TimeoutError + + monkeypatch.setattr(_grep, "check_timeout", always_timeout) + result = grep_all(GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path)) + + assert result.truncated is True + assert result.truncated_reason == "timeout" + assert result.errors[0].code == "TIMEOUT" + # Path is None on the outer timeout (no file in flight). + assert result.errors[0].path is None + + +def test_grep_all_fixed_ignore_case_word_and_invert(tmp_path): + """Literal, ignore-case, word, and invert modes are supported.""" + path = tmp_path / "data.txt" + path.write_text("alpha\ncatalog\nALPHA\n", encoding="utf-8") + + word_result = grep_all( + GrepOptions( + pattern="alpha", + paths=["*.txt"], + base=tmp_path, + fixed_string=True, + ignore_case=True, + word=True, + ) + ) + invert_result = grep_all( + GrepOptions(pattern="alpha", paths=["*.txt"], base=tmp_path, fixed_string=True, invert=True) + ) + + assert [match.line_number for match in word_result.results] == [1, 3] + assert [match.content for match in invert_result.results] == ["catalog", "ALPHA"] + + +def test_grep_all_bad_regex_is_error_result(tmp_path): + """Regex compile errors are reported instead of raised.""" + result = grep_all(GrepOptions(pattern="[", paths=["*.txt"], base=tmp_path)) + assert result.results == [] + assert result.errors[0].code == "REGEX" + + +def test_grep_all_binary_skip_and_text_override(tmp_path): + """Binary files are skipped by default and searchable with text=True.""" + binary = tmp_path / "binary.bin" + binary.write_bytes(b"\x00TODO\x00") + + skipped = grep_all(GrepOptions(pattern="TODO", paths=["*.bin"], base=tmp_path)) + as_text = grep_all(GrepOptions(pattern="TODO", paths=["*.bin"], base=tmp_path, text=True)) + + assert skipped.results == [] + assert skipped.errors[0].code == "BINARY" + assert as_text.results[0].content == "\x00TODO\x00" + + +def test_grep_all_limit_max_count_and_max_bytes(tmp_path): + """Grep reports truncation reasons for result and byte limits.""" + path = tmp_path / "many.txt" + path.write_text("TODO one\nTODO two\n", encoding="utf-8") + + limited = grep_all(GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path, limit=1)) + max_counted = grep_all(GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path, max_count=1)) + byte_limited = grep_all( + GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path, max_bytes=1) + ) + + assert limited.truncated_reason == "limit" + assert max_counted.truncated_reason == "max_count" + assert byte_limited.truncated_reason == "max_bytes" + + +def test_grep_all_max_bytes_checks_size_before_read(monkeypatch, tmp_path): + """max_bytes prevents opening a file that already exceeds the cap.""" + path = tmp_path / "large.txt" + path.write_text("TODO\n", encoding="utf-8") + + def fail_open(_self, *_args, **_kwargs): + raise AssertionError("Path.open should not be called when max_bytes trips first") + + monkeypatch.setattr(Path, "open", fail_open) + result = grep_all(GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path, max_bytes=1)) + + assert result.truncated is True + assert result.truncated_reason == "max_bytes" + + +def test_grep_all_strict_base_rejects_symlink_escape(tmp_path): + """Grep does not read linked dirs that resolve outside strict base.""" + base = tmp_path / "base" + base.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + target = outside / "secret.txt" + target.write_text("TODO\n", encoding="utf-8") + try: + (base / "linked").symlink_to(outside, target_is_directory=True) + except (OSError, NotImplementedError): # pragma: no cover - Windows-only + pytest.skip("symlinks not permitted") + + result = grep_all(GrepOptions(pattern="TODO", paths=["*.txt"], base=base, follow_symlinks=True)) + + assert result.results == [] + assert result.errors[0].code == "PERM" + + +def test_grep_all_strict_base_can_suppress_error(tmp_path): + """Grep strict-base errors respect include_errors=False.""" + base = tmp_path / "base" + base.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + (outside / "secret.txt").write_text("TODO\n", encoding="utf-8") + try: + (base / "linked").symlink_to(outside, target_is_directory=True) + except (OSError, NotImplementedError): # pragma: no cover - Windows-only + pytest.skip("symlinks not permitted") + + result = grep_all( + GrepOptions( + pattern="TODO", + paths=["*.txt"], + base=base, + follow_symlinks=True, + include_errors=False, + ) + ) + + assert result.results == [] + assert result.errors == [] + + +def test_grep_all_unreadable_file(monkeypatch, tmp_path): + """Read errors are collected as ErrorInfo records.""" + path = tmp_path / "bad.txt" + path.write_text("TODO\n", encoding="utf-8") + real_open = Path.open + + def boom(self, *args, **kwargs): + if self == path: + raise PermissionError("nope") + return real_open(self, *args, **kwargs) + + monkeypatch.setattr(Path, "open", boom) + result = grep_all(GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path)) + + assert result.errors[0].code == "UNREADABLE" + + +def test_grep_all_unstatable_file(monkeypatch, tmp_path): + """Stat errors are collected before file reads.""" + path = tmp_path / "bad.txt" + path.write_text("TODO\n", encoding="utf-8") + real_stat = Path.stat + + def boom(self, *args, **kwargs): + if self == path: + raise PermissionError("no stat") + return real_stat(self, *args, **kwargs) + + monkeypatch.setattr(Path, "stat", boom) + result = grep_all(GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path)) + + assert result.errors[0].code == "UNREADABLE" + + +def test_grep_all_unstatable_file_can_suppress_error(monkeypatch, tmp_path): + """Stat errors respect include_errors=False.""" + path = tmp_path / "bad.txt" + path.write_text("TODO\n", encoding="utf-8") + real_stat = Path.stat + + def boom(self, *args, **kwargs): + if self == path: + raise PermissionError("no stat") + return real_stat(self, *args, **kwargs) + + monkeypatch.setattr(Path, "stat", boom) + result = grep_all( + GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path, include_errors=False) + ) + + assert result.errors == [] + + +def test_grep_all_can_suppress_unreadable_and_binary_errors(monkeypatch, tmp_path): + """include_errors=False suppresses read and binary errors.""" + bad = tmp_path / "bad.txt" + bad.write_text("TODO\n", encoding="utf-8") + binary = tmp_path / "binary.bin" + binary.write_bytes(b"\x00TODO\x00") + real_open = Path.open + + def boom(self, *args, **kwargs): + if self == bad: + raise PermissionError("nope") + return real_open(self, *args, **kwargs) + + monkeypatch.setattr(Path, "open", boom) + + unreadable = grep_all( + GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path, include_errors=False) + ) + skipped_binary = grep_all( + GrepOptions(pattern="TODO", paths=["*.bin"], base=tmp_path, include_errors=False) + ) + + assert unreadable.errors == [] + assert skipped_binary.errors == [] + + +def test_grep_all_timeout_can_suppress_error(monkeypatch, tmp_path): + """File-level timeout truncation respects include_errors=False.""" + (tmp_path / "notes.txt").write_text("TODO\n", encoding="utf-8") + ticks = iter([0.0, 1.0]) + monkeypatch.setattr("rglob.agent._runtime.time.monotonic", lambda: next(ticks)) + + result = grep_all( + GrepOptions( + pattern="TODO", + paths=["*.txt"], + base=tmp_path, + timeout_seconds=0.1, + include_errors=False, + ) + ) + + assert result.truncated_reason == "timeout" + assert result.errors == [] + + +def test_grep_all_line_timeout_reports_path(monkeypatch, tmp_path): + """Line-level timeout reports the file being processed.""" + path = tmp_path / "notes.txt" + path.write_text("TODO\n", encoding="utf-8") + ticks = iter([0.0, 0.05, 1.0]) + monkeypatch.setattr("rglob.agent._runtime.time.monotonic", lambda: next(ticks)) + + result = grep_all( + GrepOptions(pattern="TODO", paths=["*.txt"], base=tmp_path, timeout_seconds=0.1) + ) + + assert result.truncated_reason == "timeout" + assert result.errors[0].code == "TIMEOUT" + assert result.errors[0].path == path + + +def test_grep_all_line_timeout_can_suppress_error(monkeypatch, tmp_path): + """Line-level timeout respects include_errors=False.""" + path = tmp_path / "notes.txt" + path.write_text("TODO\n", encoding="utf-8") + ticks = iter([0.0, 0.05, 1.0]) + monkeypatch.setattr("rglob.agent._runtime.time.monotonic", lambda: next(ticks)) + + result = grep_all( + GrepOptions( + pattern="TODO", + paths=["*.txt"], + base=tmp_path, + timeout_seconds=0.1, + include_errors=False, + ) + ) + + assert result.truncated_reason == "timeout" + assert result.errors == [] + + +def test_count_all_counts_files_lines_and_bytes(tmp_path): + """CountOptions produce structured Stats.""" + (tmp_path / "a.py").write_text("x\n\n# comment\ny\n", encoding="utf-8") + (tmp_path / "b.py").write_text("z\n", encoding="utf-8") + + result = count_all( + CountOptions(patterns=["*.py"], base=tmp_path, no_empty=True, no_comments=True) + ) + + assert result.files == 2 + assert result.lines == 3 + assert result.bytes > 0 + + +def test_count_all_limit_max_bytes_and_unreadable(monkeypatch, tmp_path): + """Count truncation and unreadable files are reported.""" + a = tmp_path / "a.py" + a.write_text("x\n", encoding="utf-8") + b = tmp_path / "b.py" + b.write_text("y\n", encoding="utf-8") + + limited = count_all(CountOptions(patterns=["*.py"], base=tmp_path, limit=1)) + byte_limited = count_all(CountOptions(patterns=["*.py"], base=tmp_path, max_bytes=1)) + + real_read = Path.read_bytes + + def boom(self): + if self == a: + raise PermissionError("nope") + return real_read(self) + + monkeypatch.setattr(Path, "read_bytes", boom) + unreadable = count_all(CountOptions(patterns=["*.py"], base=tmp_path)) + + assert limited.truncated_reason == "limit" + assert byte_limited.truncated_reason == "max_bytes" + assert unreadable.errors[0].code == "UNREADABLE" + + +def test_count_all_unstatable_file(monkeypatch, tmp_path): + """Stat errors are collected before file reads.""" + path = tmp_path / "bad.py" + path.write_text("x\n", encoding="utf-8") + real_stat = Path.stat + + def boom(self, *args, **kwargs): + if self == path: + raise PermissionError("no stat") + return real_stat(self, *args, **kwargs) + + monkeypatch.setattr(Path, "stat", boom) + result = count_all(CountOptions(patterns=["*.py"], base=tmp_path)) + + assert result.errors[0].code == "UNREADABLE" + + +def test_count_all_unstatable_file_can_suppress_error(monkeypatch, tmp_path): + """Stat errors respect include_errors=False.""" + path = tmp_path / "bad.py" + path.write_text("x\n", encoding="utf-8") + real_stat = Path.stat + + def boom(self, *args, **kwargs): + if self == path: + raise PermissionError("no stat") + return real_stat(self, *args, **kwargs) + + monkeypatch.setattr(Path, "stat", boom) + result = count_all(CountOptions(patterns=["*.py"], base=tmp_path, include_errors=False)) + + assert result.errors == [] + + +def test_count_all_max_bytes_checks_size_before_read(monkeypatch, tmp_path): + """max_bytes prevents reading a file that already exceeds the cap.""" + path = tmp_path / "large.py" + path.write_text("x\n", encoding="utf-8") + + def fail_read(_self): + raise AssertionError("read_bytes should not be called") + + monkeypatch.setattr(Path, "read_bytes", fail_read) + result = count_all(CountOptions(patterns=["*.py"], base=tmp_path, max_bytes=1)) + + assert result.truncated is True + assert result.truncated_reason == "max_bytes" + + +def test_count_all_strict_base_rejects_symlink_escape(tmp_path): + """Count does not read linked dirs that resolve outside strict base.""" + base = tmp_path / "base" + base.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + target = outside / "secret.py" + target.write_text("x\n", encoding="utf-8") + try: + (base / "linked").symlink_to(outside, target_is_directory=True) + except (OSError, NotImplementedError): # pragma: no cover - Windows-only + pytest.skip("symlinks not permitted") + + result = count_all(CountOptions(patterns=["*.py"], base=base, follow_symlinks=True)) + + assert result.files == 0 + assert result.errors[0].code == "PERM" + + +def test_count_all_strict_base_can_suppress_error(tmp_path): + """Count strict-base errors respect include_errors=False.""" + base = tmp_path / "base" + base.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + (outside / "secret.py").write_text("x\n", encoding="utf-8") + try: + (base / "linked").symlink_to(outside, target_is_directory=True) + except (OSError, NotImplementedError): # pragma: no cover - Windows-only + pytest.skip("symlinks not permitted") + + result = count_all( + CountOptions( + patterns=["*.py"], + base=base, + follow_symlinks=True, + include_errors=False, + ) + ) + + assert result.files == 0 + assert result.errors == [] + + +def test_count_all_can_suppress_unreadable_errors(monkeypatch, tmp_path): + """include_errors=False suppresses read errors.""" + path = tmp_path / "a.py" + path.write_text("x\n", encoding="utf-8") + + def boom(_self): + raise PermissionError("nope") + + monkeypatch.setattr(Path, "read_bytes", boom) + result = count_all(CountOptions(patterns=["*.py"], base=tmp_path, include_errors=False)) + + assert result.errors == [] + + +def test_count_all_timeout_can_suppress_error(monkeypatch, tmp_path): + """File-level timeout truncation respects include_errors=False.""" + (tmp_path / "a.py").write_text("x\n", encoding="utf-8") + ticks = iter([0.0, 1.0]) + monkeypatch.setattr("rglob.agent._runtime.time.monotonic", lambda: next(ticks)) + + result = count_all( + CountOptions( + patterns=["*.py"], + base=tmp_path, + timeout_seconds=0.1, + include_errors=False, + ) + ) + + assert result.truncated_reason == "timeout" + assert result.errors == [] + + +def test_count_all_line_timeout_reports_path(monkeypatch, tmp_path): + """Line-level timeout reports the file being processed.""" + path = tmp_path / "a.py" + path.write_text("x\n", encoding="utf-8") + ticks = iter([0.0, 0.05, 1.0]) + monkeypatch.setattr("rglob.agent._runtime.time.monotonic", lambda: next(ticks)) + + result = count_all(CountOptions(patterns=["*.py"], base=tmp_path, timeout_seconds=0.1)) + + assert result.truncated_reason == "timeout" + assert result.errors[0].code == "TIMEOUT" + assert result.errors[0].path == path + + +def test_count_all_line_timeout_can_suppress_error(monkeypatch, tmp_path): + """Line-level timeout respects include_errors=False.""" + path = tmp_path / "a.py" + path.write_text("x\n", encoding="utf-8") + ticks = iter([0.0, 0.05, 1.0]) + monkeypatch.setattr("rglob.agent._runtime.time.monotonic", lambda: next(ticks)) + + result = count_all( + CountOptions( + patterns=["*.py"], + base=tmp_path, + timeout_seconds=0.1, + include_errors=False, + ) + ) + + assert result.truncated_reason == "timeout" + assert result.errors == [] diff --git a/tests/test_legacy_api.py b/tests/test_legacy_api.py new file mode 100644 index 0000000..3f77649 --- /dev/null +++ b/tests/test_legacy_api.py @@ -0,0 +1,134 @@ +"""Ports of the Behave scenarios in `features/rglob.feature` to pytest. + +Each scenario in the feature file corresponds to one (or a few) test functions +here. Helpers live on `TreeBuilder` in `conftest.py` so the BDD steps and +these tests share assertions. +""" + +from __future__ import annotations + +import math + +import pytest + +import rglob + + +def test_empty_root_finds_zero_dirs(tree_builder): + """Fresh root has no `*_rglob` matches.""" + matches = rglob.rglob(str(tree_builder.root), "*_rglob") + assert matches == [] + + +def test_create_10_subdirs(tree_builder): + """10 subdirs created → 10 matches under root.""" + tree_builder.make_subdirs(10) + matches = rglob.rglob(str(tree_builder.root), "*_rglob") + assert len(matches) == 10 + + +def test_create_110_subdirs(tree_builder): + """10 top-level + 10x10 nested = 110 matches.""" + tree_builder.make_subdirs(10) + tree_builder.make_subdirs(10) + matches = rglob.rglob(str(tree_builder.root), "*_rglob") + assert len(matches) == 110 + + +def test_create_1100_subdirs(tree_builder): + """100 + 100x10 = 1100 matches.""" + tree_builder.make_subdirs(100) + tree_builder.make_subdirs(10) + matches = rglob.rglob(str(tree_builder.root), "*_rglob") + assert len(matches) == 1100 + + +def test_create_1100_text_files_size_matches(tree_builder): + """File-count and total-size assertions on a 1100-file tree.""" + tree_builder.make_subdirs(100) + tree_builder.make_subdirs(10) + tree_builder.make_files(1, ".txt") + # 1100 *_rglob dirs, 1100 .txt files + assert len(rglob.rglob(str(tree_builder.root), "*_rglob")) == 1100 + assert len(rglob.rglob(str(tree_builder.root), "*.txt")) == 1100 + + known_kib = rglob.kilobytes(tree_builder.total_known_size(".txt")) + found_kib = rglob.tsize(str(tree_builder.root), "*.txt", rglob.kilobytes) + assert math.isclose(known_kib, found_kib, rel_tol=0, abs_tol=1e-9) + + +def test_count_lines_in_text_files(tree_builder): + """10 subdirs x 2 files x 5 lines = 100 lines.""" + tree_builder.make_subdirs(10) + tree_builder.make_files(2, ".txt") + tree_builder.write_lines(".txt", lines=5) + assert rglob.lcount(str(tree_builder.root), "*.txt") == 100 + + +def test_rglob_underscore_uses_cwd(tree_builder, monkeypatch): + """`rglob_` walks from CWD; chdir to the synthetic root and check counts.""" + tree_builder.make_subdirs(10) + tree_builder.make_files(2, ".py") + monkeypatch.chdir(tree_builder.root) + assert len(rglob.rglob_("*.py")) == 20 + + +@pytest.mark.parametrize( + ("converter", "factor"), + [ + (rglob.kilobytes, 2**10), + (rglob.megabytes, 2**20), + (rglob.gigabytes, 2**30), + (rglob.terabytes, 2**40), + ], +) +def test_unit_helpers(converter, factor): + """Each unit helper divides by the right binary prefix.""" + assert converter(float(factor)) == 1.0 + assert converter(2.0 * factor) == 2.0 + + +def test_lcount_with_filter(tree_builder): + """`lcount` respects the per-line predicate.""" + tree_builder.make_subdirs(2) + tree_builder.make_files(1, ".txt", content="alpha\nbeta\n# comment\n\n") + total = rglob.lcount(str(tree_builder.root), "*.txt") + non_empty_non_comment = rglob.lcount( + str(tree_builder.root), + "*.txt", + lambda line: bool(line.strip()) and not line.lstrip().startswith("#"), + ) + assert total == 8 # 4 lines x 2 files + assert non_empty_non_comment == 4 # 2 valid lines x 2 files + + +def test_tsize_default_unit_is_megabytes(tree_builder): + """`tsize` defaults to MiB conversion.""" + tree_builder.make_subdirs(1) + tree_builder.make_files(1, ".bin", content="x" * 1024) + mb = rglob.tsize(str(tree_builder.root), "*.bin") + assert 0 < mb < 1.0 # 1 KiB → ~0.001 MiB + + +def test_tsize_skips_directories(tree_builder): + """`tsize` ignores directories matched by the glob.""" + # Build a tree with both files AND subdirs at the same level matched by `*_rglob` + tree_builder.make_subdirs(3) + tree_builder.make_files(1, "_rglob_data", content="hello") + # `*_rglob*` should match both subdir names and file names; _sum must skip the dirs. + mb = rglob.tsize(str(tree_builder.root), "*_rglob*", rglob.kilobytes) + assert mb > 0 + + +def test_module_main_smoke(tmp_path, monkeypatch): + """The `__main__` block in rglob.rglob exits cleanly for smoke purposes.""" + # We exercise the same code path the __main__ block would via direct call. + monkeypatch.chdir(tmp_path) + py = tmp_path / "x.py" + py.write_text("a = 1\n# comment\n\nb = 2\n", encoding="utf-8") + + def keep(line): + return bool(line.strip()) and not line.strip().startswith("#") + + assert rglob.lcount(str(tmp_path), "*.py", keep) == 2 + assert py.exists() diff --git a/tests/test_mcp.py b/tests/test_mcp.py new file mode 100644 index 0000000..49952ed --- /dev/null +++ b/tests/test_mcp.py @@ -0,0 +1,135 @@ +"""Tests for the optional MCP server wrapper.""" + +from __future__ import annotations + +import asyncio +import sys +import types +from collections.abc import Awaitable, Callable + +from typer.testing import CliRunner + +from rglob.agent import mcp as mcp_module +from rglob.cli import app + +runner = CliRunner(env={"COLUMNS": "200"}) + + +class FakeServer: + """Tiny stand-in for the official MCP Server.""" + + def __init__(self, name: str) -> None: + self.name = name + self.tools: dict[str, Callable[..., Awaitable[object]]] = {} + self.ran = False + + def tool(self): + """Return a decorator that records tool functions.""" + + def _register(func): + self.tools[func.__name__] = func + return func + + return _register + + async def run_stdio_async(self) -> None: + """Record that the server was run.""" + self.ran = True + + +def test_create_server_registers_tools(tmp_path): + """The MCP server exposes the planned tool list.""" + server = mcp_module.create_server(FakeServer) + assert set(server.tools) == { + "find_files", + "grep_content", + "count_lines", + "find_duplicate_files", + "describe_subcommand", + } + + (tmp_path / "a.py").write_text("TODO\n", encoding="utf-8") + find_payload = asyncio.run(server.tools["find_files"]("*.py", base=str(tmp_path))) + grep_payload = asyncio.run( + server.tools["grep_content"]("TODO", paths=["*.py"], base=str(tmp_path)) + ) + count_payload = asyncio.run(server.tools["count_lines"]("*.py", base=str(tmp_path))) + describe_payload = asyncio.run(server.tools["describe_subcommand"]("find")) + + assert find_payload["results"][0]["relative_path"] == "a.py" + assert grep_payload["results"][0]["content"] == "TODO" + assert count_payload["files"] == 1 + assert describe_payload["name"] == "find" + + +def test_find_duplicate_files_tool(tmp_path): + """The duplicate MCP tool returns DuplicateSearchResult payloads.""" + (tmp_path / "a.bin").write_bytes(b"same") + (tmp_path / "b.bin").write_bytes(b"same") + server = mcp_module.create_server(FakeServer) + + payload = asyncio.run(server.tools["find_duplicate_files"]("*.bin", base=str(tmp_path))) + + assert len(payload["results"]) == 1 + + +def test_load_server_factory_missing_extra(monkeypatch): + """If `mcp` isn't installed, the factory raises with a clear message.""" + # Make every relevant `mcp.*` module appear absent. Using sys.modules + # lets us simulate the un-installed state without actually uninstalling + # the optional dep from the test venv. + for name in list(sys.modules): + if name == "mcp" or name.startswith("mcp."): + monkeypatch.delitem(sys.modules, name) + import builtins + + real_import = builtins.__import__ + + def _block_mcp(name, *args, **kwargs): + if name.startswith("mcp"): + raise ModuleNotFoundError(f"No module named {name!r}") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", _block_mcp) + + import pytest + + with pytest.raises(RuntimeError, match="requires the optional mcp extra"): + mcp_module._load_server_factory() + + +def test_load_server_factory_success(monkeypatch): + """The lazy MCP import accepts a module exposing FastMCP.""" + fake_mcp = types.ModuleType("mcp") + fake_server_pkg = types.ModuleType("mcp.server") + fake_fastmcp_module = types.ModuleType("mcp.server.fastmcp") + fake_fastmcp_module.FastMCP = FakeServer + monkeypatch.setitem(sys.modules, "mcp", fake_mcp) + monkeypatch.setitem(sys.modules, "mcp.server", fake_server_pkg) + monkeypatch.setitem(sys.modules, "mcp.server.fastmcp", fake_fastmcp_module) + + factory = mcp_module._load_server_factory() + + assert factory("rglob").name == "rglob" + + +def test_main_runs_stdio(monkeypatch): + """main() runs the server's stdio coroutine.""" + fake = FakeServer("rglob") + monkeypatch.setattr(mcp_module, "create_server", lambda: fake) + + mcp_module.main() + + assert fake.ran is True + + +def test_cli_mcp_without_extra_is_json_error(monkeypatch): + """Without the optional SDK, `rglob mcp` fails as a stable JSON envelope.""" + + def boom() -> None: + raise RuntimeError("rglob mcp requires the optional mcp extra") + + monkeypatch.setattr(mcp_module, "main", boom) + result = runner.invoke(app, ["mcp"]) + assert result.exit_code == 2 + assert "UNSUPPORTED_PLATFORM" in result.stdout diff --git a/tests/test_mcp_integration.py b/tests/test_mcp_integration.py new file mode 100644 index 0000000..fbdeb3b --- /dev/null +++ b/tests/test_mcp_integration.py @@ -0,0 +1,150 @@ +"""Real-protocol integration test for `rglob mcp`. + +Spawns the actual `rglob mcp` subprocess and drives it through the +official `mcp` SDK's stdio client. This catches breakage that mocked +unit tests cannot — for example, importing the wrong server class +(`mcp.server.Server` vs `mcp.server.fastmcp.FastMCP`), schema- +serialization issues over the wire, or async-loop bugs in the entry +point. + +The test is skipped when the optional `[mcp]` extra is not installed +so the rest of the suite remains runnable on a minimal env. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +mcp_client = pytest.importorskip("mcp.client.stdio") +mcp_session = pytest.importorskip("mcp") + +from mcp import ClientSession, StdioServerParameters # noqa: E402 +from mcp.client.stdio import stdio_client # noqa: E402 + + +def _server_params() -> StdioServerParameters: + """Build the subprocess invocation for the rglob MCP server.""" + return StdioServerParameters( + command=sys.executable, + args=["-m", "rglob.cli", "mcp"], + env=None, + ) + + +@pytest.mark.asyncio +async def test_mcp_lists_documented_tools(tmp_path: Path) -> None: + """A real MCP `tools/list` request returns the five documented tools.""" + async with ( + stdio_client(_server_params()) as (read, write), + ClientSession(read, write) as session, + ): + await session.initialize() + tools = await session.list_tools() + names = {tool.name for tool in tools.tools} + + assert names >= { + "find_files", + "grep_content", + "count_lines", + "find_duplicate_files", + "describe_subcommand", + } + + +@pytest.mark.asyncio +async def test_mcp_find_files_returns_search_result(tmp_path: Path) -> None: + """`find_files` over a real subprocess returns FileSearchResult-shaped JSON.""" + (tmp_path / "alpha.py").write_text("a\n", encoding="utf-8") + (tmp_path / "beta.py").write_text("b\n", encoding="utf-8") + (tmp_path / "gamma.txt").write_text("g\n", encoding="utf-8") + + async with ( + stdio_client(_server_params()) as (read, write), + ClientSession(read, write) as session, + ): + await session.initialize() + result = await session.call_tool( + "find_files", + {"pattern": "*.py", "base": str(tmp_path)}, + ) + + payload = _structured_payload(result) + assert isinstance(payload, dict) + assert {"results", "truncated", "total_files_searched"} <= set(payload) + names = sorted(Path(match["path"]).name for match in payload["results"]) + assert names == ["alpha.py", "beta.py"] + + +@pytest.mark.asyncio +async def test_mcp_grep_content_returns_line_search_result(tmp_path: Path) -> None: + """`grep_content` returns LineSearchResult records over the wire.""" + (tmp_path / "notes.txt").write_text("TODO one\nfine\nTODO two\n", encoding="utf-8") + + async with ( + stdio_client(_server_params()) as (read, write), + ClientSession(read, write) as session, + ): + await session.initialize() + result = await session.call_tool( + "grep_content", + {"pattern": "TODO", "paths": ["*.txt"], "base": str(tmp_path)}, + ) + + payload = _structured_payload(result) + assert isinstance(payload, dict) + contents = sorted(match["content"] for match in payload["results"]) + assert contents == ["TODO one", "TODO two"] + + +@pytest.mark.asyncio +async def test_mcp_describe_subcommand_returns_manifest() -> None: + """`describe_subcommand` returns the same manifest the CLI emits.""" + async with ( + stdio_client(_server_params()) as (read, write), + ClientSession(read, write) as session, + ): + await session.initialize() + result = await session.call_tool( + "describe_subcommand", + {"name": "find"}, + ) + + payload = _structured_payload(result) + assert isinstance(payload, dict) + assert payload["name"] == "find" + assert payload["agent_api_version"] == "1.0" + assert "schemas" in payload + + +def _structured_payload(call_result: object) -> object: + """Extract the JSON-serialisable payload from an MCP CallToolResult. + + The `mcp` SDK exposes a tool result as either a list of content blocks + (older versions) or a `structured_content` field (newer versions). + Older versions return a `TextContent` whose `.text` is a JSON string; + we parse that to get the same dict shape. + """ + import json + + # `structuredContent` is FastMCP/MCP SDK's camelCase Pydantic field; + # newer SDKs may also expose `structured_content` as an alias. + structured = getattr(call_result, "structuredContent", None) + if structured is None: + structured = getattr(call_result, "structured_content", None) + if structured is not None: + # FastMCP wraps non-Pydantic-model returns in {"result": } + # because the MCP spec requires structured content to be a JSON + # object. Unwrap so callers see the underlying SearchResult shape. + if isinstance(structured, dict) and set(structured) == {"result"}: + return structured["result"] + return structured + content = getattr(call_result, "content", None) + if content: + first = content[0] + text = getattr(first, "text", None) + if text is not None: + return json.loads(text) + raise AssertionError(f"unexpected MCP result shape: {call_result!r}") diff --git a/tests/test_properties.py b/tests/test_properties.py new file mode 100644 index 0000000..9277fe9 --- /dev/null +++ b/tests/test_properties.py @@ -0,0 +1,118 @@ +"""Hypothesis property tests for the modern `find()` API. + +Properties tested: +- Cardinality: `find(p, "*")` enumerates everything under `p` exactly once, + matching a brute-force `os.walk` baseline. +- Monotonicity: deeper `max_depth` never returns *fewer* results than a + shallower one. +- Exclude commutativity: excluding `[A, B]` equals excluding `[B, A]`. +- Case-insensitive ⊇ case-sensitive: with `case_sensitive=False` the + result set is a superset of the case-sensitive one. +""" + +from __future__ import annotations + +import os +import string +from pathlib import Path + +from hypothesis import given, settings +from hypothesis import strategies as st + +from rglob import find_all + +# Safe directory / filename strategies — restricted to ASCII alphanumerics +# so we don't depend on FS-encoding quirks on macOS/Windows. +_safe_name = st.text( + alphabet=string.ascii_lowercase + string.digits + "_", + min_size=1, + max_size=8, +).filter(lambda s: not s.startswith(".")) + + +def _build_tree(root: Path, spec: list[tuple[str, ...]]) -> None: + """Create files/dirs from a spec of path tuples. + + Skips entries that would collide with an already-created file (e.g. + `[('a',), ('a', 'b')]` — second entry needs `a/` as a directory, but + `a` is already a file). Hypothesis explores both orderings, and + skipping keeps the property invariants safe to assert. + """ + for parts in spec: + if not parts: + continue + *dir_parts, leaf = parts + d = root + skip = False + for part in dir_parts: + d = d / part + if d.exists() and not d.is_dir(): + skip = True + break + d.mkdir(exist_ok=True) + if skip: + continue + leaf_path = d / leaf + if leaf_path.exists() and leaf_path.is_dir(): + continue # leaf already a dir — would collide + leaf_path.write_text("x") + + +_path_spec = st.lists( + st.lists(_safe_name, min_size=1, max_size=4), + min_size=0, + max_size=12, + unique_by=tuple, +).map(lambda lst: [tuple(p) for p in lst]) + + +@given(spec=_path_spec) +@settings(max_examples=40, deadline=None) +def test_find_star_matches_oswalk(tmp_path_factory, spec): + """`find(p, "*")` enumerates the same set as a brute-force os.walk.""" + root = tmp_path_factory.mktemp("hyp") + _build_tree(root, spec) + + walked: set[Path] = set() + for dirpath, dirnames, filenames in os.walk(root): + for name in dirnames + filenames: + walked.add(Path(dirpath) / name) + + found = set(find_all(root, "*")) + assert found == walked + + +@given(spec=_path_spec, d=st.integers(min_value=0, max_value=6)) +@settings(max_examples=40, deadline=None) +def test_max_depth_is_monotonic(tmp_path_factory, spec, d): + """Increasing `max_depth` never drops a previously-found path.""" + root = tmp_path_factory.mktemp("hyp") + _build_tree(root, spec) + + shallow = set(find_all(root, "*", max_depth=d)) + deep = set(find_all(root, "*", max_depth=d + 2)) + assert shallow <= deep + + +@given(spec=_path_spec) +@settings(max_examples=20, deadline=None) +def test_exclude_is_commutative(tmp_path_factory, spec): + """Order of exclude patterns must not affect the result.""" + root = tmp_path_factory.mktemp("hyp") + _build_tree(root, spec) + + ab = set(find_all(root, "*", exclude=["*1*", "*2*"])) + ba = set(find_all(root, "*", exclude=["*2*", "*1*"])) + assert ab == ba + + +@given(spec=_path_spec) +@settings(max_examples=20, deadline=None) +def test_case_insensitive_superset(tmp_path_factory, spec): + """Case-insensitive matching is a superset of case-sensitive matching.""" + root = tmp_path_factory.mktemp("hyp") + _build_tree(root, spec) + + sensitive = set(find_all(root, "*", case_sensitive=True)) + insensitive = set(find_all(root, "*", case_sensitive=False)) + assert sensitive <= insensitive