From 35c5eb7ef493ef536d05fc415b0126c32362e977 Mon Sep 17 00:00:00 2001 From: FBruzzesi Date: Sun, 26 Jul 2026 19:30:54 +0200 Subject: [PATCH 1/3] RFC, docs: Add AGENTS.md --- .github/workflows/check_docs_build.yml | 9 +- AGENTS.md | 206 +++++++++++++++++++++++++ Makefile | 7 + 3 files changed, 215 insertions(+), 7 deletions(-) create mode 100644 AGENTS.md diff --git a/.github/workflows/check_docs_build.yml b/.github/workflows/check_docs_build.yml index 0514f06b37..a119462ff0 100644 --- a/.github/workflows/check_docs_build.yml +++ b/.github/workflows/check_docs_build.yml @@ -30,10 +30,5 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV - - name: Install dependencies - run: uv pip install -e ".[dask,ibis]" --group docs - - name: Run hooks manually - run: | - python utils/generate_backend_completeness.py - python utils/generate_zen_content.py - - run: zensical build --strict + - name: Check docs build + run: docs-build diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..0066890f53 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,206 @@ +# AGENTS.md + +Narwhals is an extremely lightweight, zero-dependency compatibility layer between Python dataframe libraries. + +It lets library authors write dataframe-agnostic code once using a subset of the Polars API, +and have it work across pandas, Polars, PyArrow, cuDF, Modin, Dask, DuckDB, PySpark, Ibis, and SQLFrame, +without depending on any of them. + +The primary audience is **library maintainers**, not end users. +Because of that, stability and backwards compatibility are taken extremely seriously. + +## Read the docs first + +Almost everything an agent needs is already documented. Read the relevant page instead of inferring +from the source, and update the page when you change the behaviour it describes. + +| Topic | Read | +| --- | --- | +| Internal architecture: expressions, nodes, expression metadata, broadcasting, `over` push-down, group-by | [docs/how_it_works.md](docs/how_it_works.md) | +| Contributor workflow: env setup, test invocations, backend-specific rules, docstring style, PR conventions | [CONTRIBUTING.md](CONTRIBUTING.md) | +| Stable API guarantees and the `main` vs `stable.v1` / `stable.v2` diff | [docs/backcompat.md](docs/backcompat.md) | +| Adding a backend: compliant protocols, plugin entry points, the IO namespace contract | [docs/extending.md](docs/extending.md) | +| Row-order semantics: what `DataFrame` guarantees, what `LazyFrame` doesn't, `over(order_by=...)` | [docs/concepts/order_dependence.md](docs/concepts/order_dependence.md) | +| Null vs NaN: which methods exist for which, and what pandas muddies | [docs/concepts/null_handling.md](docs/concepts/null_handling.md) | +| Why the pandas `group_by` `UserWarning` exists and how to avoid triggering it | [docs/concepts/improve_group_by_operation.md](docs/concepts/improve_group_by_operation.md) | +| Boolean semantics, the pandas index, non-string column names | [docs/concepts/](docs/concepts/) | +| Which methods each backend implements | [docs/api-completeness/](docs/api-completeness/) (generated, do not hand-edit) | +| Public API surface | [docs/api-reference/](docs/api-reference/) (member lists are validated by CI) | +| `narwhals.sql`: generating SQL from Narwhals expressions | [docs/generating_sql.md](docs/generating_sql.md) | +| Security reporting and release-permission policy | [docs/security.md](docs/security.md) | + +The one-sentence summary of [docs/how_it_works.md](docs/how_it_works.md), worth internalising before +touching anything in `_pandas_like/`, `_arrow/`, or `_compliant/`: + +> An expression is a function from a DataFrame to a sequence of Series. + +## Layered design + +1. **Public API** ([src/narwhals/dataframe.py](src/narwhals/dataframe.py), + [series.py](src/narwhals/series.py), [expr.py](src/narwhals/expr.py), ...): the user-facing + Polars-like API. Thin wrappers that build `ExprNode`s and dispatch to compliant backends. +2. **Compliant wrappers** (`src/narwhals/_pandas_like/`, `_arrow/`, `_polars/`, `_duckdb/`, + `_spark_like/`, `_dask/`, `_ibis/`): each backend implements Narwhals-compliant DataFrames, + Series, Exprs, and Namespaces that translate the Polars-like API to native calls. Shared + protocols and base classes live in `src/narwhals/_compliant/`. +3. **Native libraries** (pandas, Polars, PyArrow, ...): the actual computation engines, never + directly depended on. + +## Source layout + +``` +src/narwhals/ + _pandas_like/ # Compliant layer for pandas, Modin, cuDF (fireducks is silently allowed here + # as a pandas drop-in; see `IMPORT_HOOKS` in dependencies.py) + _arrow/ # Compliant layer for PyArrow + _polars/ # Compliant layer for Polars + _duckdb/ # Compliant layer for DuckDB + _spark_like/ # Compliant layer for PySpark, Spark Connect, SQLFrame + _dask/ # Compliant layer for Dask + _ibis/ # Compliant layer for Ibis + _interchange/ # Interchange protocol support + _sql/ # Shared SQL generation utilities + _compliant/ # Protocols and base classes shared across backends + stable/ # Frozen stable API namespaces (v1, v2) + testing/ # Public testing utilities (e.g. asserts) + _expression_parsing.py # ExprNode, ExprMetadata, ExpansionKind, node evaluation, `over` + # push-down (entry point is `Expr._with_over_node` in expr.py) + _utils.py # Implementation, Version, and shared helpers + compliant.py # Re-exports of the protocols that backends must implement + plugins.py # Plugin system for external backends + dataframe.py # Public DataFrame/LazyFrame API + series.py # Public Series API + expr.py # Public Expr API + sql.py # Public `narwhals.sql` (SQL generation; requires DuckDB) + translate.py # from_native / to_native (re-exported via narwhals/__init__.py) + _translate.py # Conversion/structural-typing protocols (NOT from_native/to_native) + dependencies.py # Backend detection (isinstance checks without imports) +tests/ # Test suite +tpch/ # TPC-H benchmark queries +packages/ # uv workspace members (currently: test-plugin) +``` + +## Hard rules + +These are non-negotiable and the most common source of review comments. The long form, with +rationale, is in [CONTRIBUTING.md](CONTRIBUTING.md). + +* **Zero dependencies.** Narwhals must never add a runtime dependency. It only uses what the user + passes in. +* **Never import anything for `isinstance` checks.** Use the functions in + [src/narwhals/dependencies.py](src/narwhals/dependencies.py) (e.g. `is_pandas_dataframe`). +* **Never iterate over rows.** Assume infinite rows. Column iteration is acceptable. +* **Never modify user input data.** Especially with pandas: no inplace operations on user-provided + objects. +* **100% branch coverage** is enforced by the full-coverage CI job. When a branch is genuinely + unreachable (e.g. gated on an unsupported backend version), mark it `# pragma: no cover` with a + one-line reason. +* **Breaking changes never land in `narwhals.stable.v1` or `narwhals.stable.v2`.** New public APIs + land in the main `narwhals` namespace and graduate into the next stable version. See + [docs/backcompat.md](docs/backcompat.md), and add an entry to its `main` vs `stable.*` diff when + the namespaces diverge. + +Backend-specific rules (no pandas `apply`/`map`/`assign`/`drop`/`reset_index`/`rename`, no Polars +`map_elements`, no ordering assumptions or materialisation on lazy backends, DuckDB Python API over +SQL) are listed in full under +[CONTRIBUTING.md → Backend-specific considerations](CONTRIBUTING.md#backend-specific-considerations). + +## The harness: run all of this before committing + +Run these from the repo root. If you have not activated `.venv`, prefix the non-`make` commands with +`uv run`. + +**1. Lint and formatting** (also runs the repo's custom checks: docstring validation, banned +imports, API-reference sync, slotted classes, `uv.lock` freshness): + +```bash +prek run --all-files +``` + +**2. Static typing** (mypy, pyright, and pyrefly, per the `typing` target in +[Makefile](Makefile)): + +```bash +make typing +``` + +Optionally, the type-completeness gate that CI also runs: + +```bash +make typing-coverage +``` + +**3. Full test suite with 100% coverage** (this is the `pytest-full-coverage` CI job, and the one +that catches missing `# pragma: no cover`): + +```bash +PYTEST_ADDOPTS="--numprocesses=logical" make run-ci DEPS="--extra pandas --extra dask --group core-tests --group sklearn --group plugins" CMD="pytest tests --cov=src --cov=tests --cov-fail-under=100 --runslow --durations=30 --constructors=pandas,pandas[nullable],pandas[pyarrow],pyarrow,polars[eager],polars[lazy],dask,duckdb,sqlframe" +``` + +**4. Doctests** (docstring examples are executed; reprs differ across versions, so CI only runs +these on the latest Python): + +```bash +make run-ci DEPS="--extra pandas --extra dask --group core-tests --group sklearn" CMD="pytest src --doctest-modules" +``` + +**5. Docs build**, if you touched anything under `docs/` or any docstring. The build *executes* the +`exec="yes"` code blocks, so a stale snippet is a build failure: + +```bash +make docs-build +``` + +To preview instead of just building: `make docs-serve` (or `make docs-clean-serve` if it does not +refresh). Docs are built with `zensical` (configured in [zensical.toml](zensical.toml)), not mkdocs +— the nav lives there, so a new page must be added to it. + +### Faster inner loop + +Full coverage runs are slow. While iterating: + +```bash +uv run pytest tests/path/to/test_file.py # one file +uv run pytest tests --constructors=pandas,polars[eager],pyarrow +uv run pytest tests --all-cpu-constructors # needs --extra modin --extra pyspark +``` + +* Default constructors are `pandas,pandas[pyarrow],polars[eager],pyarrow,duckdb,sqlframe,ibis` + (overridable via the `NARWHALS_DEFAULT_CONSTRUCTORS` env var). +* Hypothesis tests are skipped unless you pass `--runslow`. +* Dask and Modin are not in `local-dev`; add `--extra dask --extra modin` to test them locally. + +Do not treat a green fast run as sufficient: the coverage gate and the lazy/SQL constructors +(`duckdb`, `sqlframe`, `dask`) catch a distinct class of bug, so run step 3 before committing. + +### Test failure patterns + +* `request.applymarker(pytest.mark.xfail)` — planned but not yet supported features. +* `pytest.mark.skipif` — conditional skips (e.g. version constraints). +* `pytest.raises` — expected exceptions. + +Always document the reason in a comment. Details and examples: +[CONTRIBUTING.md → Test Failure Patterns](CONTRIBUTING.md#test-failure-patterns). + +## Code style + +* Line length: 90 characters. ruff for both formatting and linting. +* Docstrings: Google style, validated by `utils/check_docstrings.py` and darglint via `prek`. + Docstring examples should import *one* dataframe library, and we deliberately balance which + backend is used across the docs. See + [CONTRIBUTING.md → Writing the doc(strings)](CONTRIBUTING.md#8-writing-the-docstrings). +* Static typing with mypy (strict), pyright, and pyrefly. +* In `_pandas_like/`, native types are typed as pandas types (the package is shared with Modin and + cuDF). In `_spark_like/`, native types are typed as SQLFrame (shared with PySpark). +* Public API changes must be reflected in `docs/api-reference/` — `prek` fails otherwise. + +## Pull requests + +Title must start with a [conventional commit](https://www.conventionalcommits.org/) type: `build`, +`chore`, `ci`, `depr`, `docs`, `feat`, `fix`, `perf`, `refactor`, `release`, `test` (append `!` for +breaking changes). The title becomes the changelog entry. + +**AI-assisted contributions must be disclosed** in the dedicated PR-template field, and the author +is accountable for every line. Read +[CONTRIBUTING.md → AI-assisted contributions](CONTRIBUTING.md#ai-assisted-contributions) before +opening a PR. diff --git a/Makefile b/Makefile index c06bedd090..cc26ab7200 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,13 @@ typing: ## Run type checkers typing-coverage: ## Run type checkers uv run --group typing pyrefly coverage check src/narwhals --public-only +.PHONY: docs-build +docs-build: ## Build the docs locally + uv run --group docs zensical build --clean + uv run --group docs --extra dask --extra ibis utils/generate_backend_completeness.py + uv run --group docs utils/generate_zen_content.py + uv run --group docs zensical build --strict + .PHONY: docs-serve docs-serve: ## Build and serve the docs locally uv run --group docs --extra dask --extra ibis utils/generate_backend_completeness.py From 405a058756064147bf5065f820990b20a5728893 Mon Sep 17 00:00:00 2001 From: FBruzzesi Date: Sun, 26 Jul 2026 19:40:13 +0200 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=A4=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/check_docs_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check_docs_build.yml b/.github/workflows/check_docs_build.yml index a119462ff0..0cc513bee3 100644 --- a/.github/workflows/check_docs_build.yml +++ b/.github/workflows/check_docs_build.yml @@ -31,4 +31,4 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV - name: Check docs build - run: docs-build + run: make docs-build From 5735064601162da50e6f229809f4a3ecb3e01e3a Mon Sep 17 00:00:00 2001 From: Edoardo Abati <29585319+EdAbati@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:39:31 +0200 Subject: [PATCH 3/3] docs: reuse Makefile targets in CI and simplify AGENTS.md (#3825) --- .github/workflows/deploy-docs.yml | 9 ++------- .github/workflows/pytest.yml | 4 ++-- AGENTS.md | 9 +++++---- CONTRIBUTING.md | 5 ++++- Makefile | 32 ++++++++++++++++++++++--------- 5 files changed, 36 insertions(+), 23 deletions(-) diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 3280138f3b..2cb4843405 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -36,13 +36,8 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV - - name: Install dependencies - run: uv pip install -e ".[dask,ibis]" --group docs - - name: Run hooks manually - run: | - python utils/generate_backend_completeness.py - python utils/generate_zen_content.py - - run: zensical build --clean + - name: Build docs + run: make docs-build - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: path: site diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 4ebb0ae7f8..3d4424056e 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -82,11 +82,11 @@ jobs: cache-suffix: pytest-full-coverage-${{ matrix.python-version }} cache-dependency-glob: "pyproject.toml" - name: Run pytest - run: make run-ci DEPS="--extra pandas --extra dask --group core-tests --group sklearn --group plugins" CMD="pytest tests --cov=src --cov=tests --cov-fail-under=100 --runslow --durations=30 --constructors=pandas,pandas[nullable],pandas[pyarrow],pyarrow,polars[eager],polars[lazy],dask,duckdb,sqlframe" + run: make test-full-coverage - name: Run doctests # reprs differ between versions, so we only run doctests on the latest Python if: matrix.python-version == '3.13' - run: make run-ci DEPS="--extra pandas --extra dask --group core-tests --group sklearn" CMD="pytest src --doctest-modules" + run: make doctest # Test against smaller dependency set, used e.g. on Gentoo. pytest-narrower-dependencies: diff --git a/AGENTS.md b/AGENTS.md index 0066890f53..0f816534bc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -134,14 +134,14 @@ make typing-coverage that catches missing `# pragma: no cover`): ```bash -PYTEST_ADDOPTS="--numprocesses=logical" make run-ci DEPS="--extra pandas --extra dask --group core-tests --group sklearn --group plugins" CMD="pytest tests --cov=src --cov=tests --cov-fail-under=100 --runslow --durations=30 --constructors=pandas,pandas[nullable],pandas[pyarrow],pyarrow,polars[eager],polars[lazy],dask,duckdb,sqlframe" +make test-full-coverage ``` **4. Doctests** (docstring examples are executed; reprs differ across versions, so CI only runs these on the latest Python): ```bash -make run-ci DEPS="--extra pandas --extra dask --group core-tests --group sklearn" CMD="pytest src --doctest-modules" +make doctest ``` **5. Docs build**, if you touched anything under `docs/` or any docstring. The build *executes* the @@ -151,8 +151,9 @@ make run-ci DEPS="--extra pandas --extra dask --group core-tests --group sklearn make docs-build ``` -To preview instead of just building: `make docs-serve` (or `make docs-clean-serve` if it does not -refresh). Docs are built with `zensical` (configured in [zensical.toml](zensical.toml)), not mkdocs +To preview instead of just building: `make docs-clean-serve` (or plain `make docs-serve` for a +quicker preview without the clean rebuild). +Docs are built with `zensical` (configured in [zensical.toml](zensical.toml)), not mkdocs — the nav lives there, so a new page must be added to it. ### Faster inner loop diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a3557dd894..8371f3eba2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,7 +14,8 @@ If you've got experience with open source contributions, the following instructi - `uv sync --group local-dev` (creates `.venv` and installs project + dev deps) - Install prek as a git hook: `uv run prek install` - To run tests: `uv run pytest` -- To run all linting checks: `make lint` +- To run ruff formatting and linting: `make lint` +- To run all pre-commit checks (which include ruff): `uv run prek run --all-files` - To run static typing checks: `make typing` For more detailed and beginner-friendly instructions, see below! @@ -332,6 +333,8 @@ The docs should refresh when you make changes. If they don't, press `ctrl+C`, an make docs-clean-serve ``` +which rebuilds everything from a clean state (via `make docs-build`) before serving. + ### 10. Pull requests When you have resolved your issue, [open a pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork) in the Narwhals repository. diff --git a/Makefile b/Makefile index cc26ab7200..2eb3c8201a 100644 --- a/Makefile +++ b/Makefile @@ -23,25 +23,39 @@ typing: ## Run type checkers typing-coverage: ## Run type checkers uv run --group typing pyrefly coverage check src/narwhals --public-only -.PHONY: docs-build -docs-build: ## Build the docs locally - uv run --group docs zensical build --clean +.PHONY: docs-dynamic-content +docs-dynamic-content: ## Regenerate the dynamic docs pages (API completeness tables, docs/this.md, ...) uv run --group docs --extra dask --extra ibis utils/generate_backend_completeness.py uv run --group docs utils/generate_zen_content.py - uv run --group docs zensical build --strict + +.PHONY: docs-build +docs-build: ## Build the docs from a clean state, failing on warnings + $(MAKE) docs-dynamic-content + uv run --group docs zensical build --clean --strict .PHONY: docs-serve -docs-serve: ## Build and serve the docs locally - uv run --group docs --extra dask --extra ibis utils/generate_backend_completeness.py - uv run --group docs utils/generate_zen_content.py +docs-serve: ## Serve the docs locally + $(MAKE) docs-dynamic-content uv run --group docs zensical serve .PHONY: docs-clean-serve docs-clean-serve: ## Rebuild docs from a clean state and serve them locally - uv run --group docs zensical build --clean - $(MAKE) docs-serve + $(MAKE) docs-build + uv run --group docs zensical serve .PHONY: run-ci run-ci: ## Print resolved deps, then run a command via uv. Usage: make run-ci DEPS="" CMD="" [RUN_ONLY=""] uv export --no-annotate --no-hashes $(DEPS) uv run $(DEPS) $(RUN_ONLY) $(CMD) + +.PHONY: doctest +doctest: ## Run doctests + make run-ci \ + DEPS="--extra pandas --extra dask --group core-tests --group sklearn" \ + CMD="pytest src --doctest-modules" + +.PHONY: test-full-coverage +test-full-coverage: ## Run the full test suite with 100% coverage across all constructors as in CI + PYTEST_ADDOPTS="--numprocesses=logical" make run-ci \ + DEPS="--extra pandas --extra dask --group core-tests --group sklearn --group plugins" \ + CMD="pytest tests --cov=src --cov=tests --cov-fail-under=100 --runslow --durations=30 --constructors=pandas,pandas[nullable],pandas[pyarrow],pyarrow,polars[eager],polars[lazy],dask,duckdb,sqlframe"