From 31d84e1fd79598d13d2c757fa840f3c0fe0274bb Mon Sep 17 00:00:00 2001 From: sidmohan0 <61345237+sidmohan0@users.noreply.github.com> Date: Mon, 27 Apr 2026 02:52:02 +0200 Subject: [PATCH] chore: clarify dependency requirement scopes --- .github/workflows/benchmark.yml | 2 +- .github/workflows/ci.yml | 9 +-- .github/workflows/release.yml | 7 +- CONTRIBUTING.md | 7 ++ README.md | 1 + justfile | 6 +- requirements-benchmark.txt | 3 + requirements-dev.txt | 12 ++- requirements-docs.txt | 2 + requirements-test.txt | 4 + requirements.txt | 26 ++----- setup.py | 128 ++++++++++++++++++-------------- tox.ini | 20 +++-- 13 files changed, 125 insertions(+), 102 deletions(-) create mode 100644 requirements-benchmark.txt create mode 100644 requirements-docs.txt create mode 100644 requirements-test.txt diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 4bac97d1..e0bc7d5c 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -27,7 +27,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -e ".[nlp]" - pip install -r requirements-dev.txt + pip install -r requirements-benchmark.txt # Verify pytest-benchmark is installed and working python -c "import pytest_benchmark; print('pytest-benchmark version:', pytest_benchmark.__version__)" python -m pytest --version diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ffb6fda..8af5182c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,26 +47,25 @@ jobs: python-version: ${{ matrix.python-version }} cache: "pip" - - name: Install base tooling + - name: Upgrade pip run: | python -m pip install --upgrade pip - pip install pytest pytest-cov coverage - name: Install dependencies (core) if: matrix.install-profile == 'core' run: | - pip install -e ".[dev,cli]" + pip install -e ".[test,cli]" -r requirements-test.txt - name: Install dependencies (nlp) if: matrix.install-profile == 'nlp' run: | - pip install -e ".[dev,cli,nlp]" + pip install -e ".[test,cli,nlp]" -r requirements-test.txt python -m spacy download en_core_web_sm - name: Install dependencies (nlp-advanced) if: matrix.install-profile == 'nlp-advanced' run: | - pip install -e ".[dev,cli,nlp,nlp-advanced]" + pip install -e ".[test,cli,nlp,nlp-advanced]" -r requirements-test.txt python -m spacy download en_core_web_sm - name: Run tests (core) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 98034d7e..53a39dfc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -126,8 +126,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e ".[all,dev]" - pip install -r requirements-dev.txt + pip install -e ".[all,test]" + pip install -r requirements-test.txt pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1.tar.gz - name: Run tests with segfault protection @@ -158,8 +158,7 @@ jobs: - name: Install core + CLI dependencies run: | python -m pip install --upgrade pip - pip install pytest pytest-cov coverage - pip install -e ".[dev,cli]" + pip install -e ".[test,cli]" -r requirements-test.txt - name: Run Python 3.13 core + CLI tests run: | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 285dd51e..483a2b3e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,6 +36,13 @@ python -m pip install --upgrade pip pip install -e ".[dev,cli]" ``` +For pinned local tooling, install the development requirements after the editable +package install: + +```bash +pip install -r requirements-dev.txt +``` + For optional NLP or OCR work, install the relevant extras: ```bash diff --git a/README.md b/README.md index e3a211b3..62f7e10d 100644 --- a/README.md +++ b/README.md @@ -162,5 +162,6 @@ cd datafog-python python -m venv .venv source .venv/bin/activate # Windows: .venv\Scripts\activate pip install -e ".[all,dev]" +pip install -r requirements-dev.txt pytest tests/ ``` diff --git a/justfile b/justfile index 2e66b4d8..34415cb4 100644 --- a/justfile +++ b/justfile @@ -21,8 +21,8 @@ default: # Set up the development environment setup: python -m venv {{venv_dir}} - {{pip}} install -e ".[dev]" - {{pip}} install isort black blacken-docs flake8 tox coverage pytest pytest-cov + {{pip}} install -e ".[dev,cli]" + {{pip}} install -r requirements-dev.txt # Format the code format: @@ -79,4 +79,4 @@ upload: clean publish: tag upload # Run all checks (format, lint, test, coverage) -check: format lint test coverage \ No newline at end of file +check: format lint test coverage diff --git a/requirements-benchmark.txt b/requirements-benchmark.txt new file mode 100644 index 00000000..b0f3fd98 --- /dev/null +++ b/requirements-benchmark.txt @@ -0,0 +1,3 @@ +# Benchmark-only dependencies. +-r requirements-test.txt +pytest-benchmark==4.0.0 diff --git a/requirements-dev.txt b/requirements-dev.txt index 7c650063..1657794c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,11 @@ -# Include the base requirements --r requirements.txt +# Local development tooling. +# +# Install the package itself with the extras you need, for example: +# pip install -e ".[dev,cli]" +# pip install -e ".[all,dev]" +-r requirements-test.txt +-r requirements-docs.txt -# Development and testing dependencies just==0.8.162 isort==5.13.2 black==26.3.1 @@ -14,4 +18,4 @@ pytest-benchmark==4.0.0 mypy==1.9.0 autoflake==2.2.1 pre-commit==3.7.0 -ruff==0.3.4 \ No newline at end of file +ruff==0.3.4 diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 00000000..d3080cf5 --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1,2 @@ +# Documentation-only dependencies. +sphinx==7.2.6 diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 00000000..3f3b434f --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,4 @@ +# Test-only dependencies. +pytest==9.0.3 +pytest-asyncio==1.3.0 +pytest-cov==7.1.0 diff --git a/requirements.txt b/requirements.txt index f4d96bc1..caa5de09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,23 +1,7 @@ -# Core dependencies with pinned versions -pandas==2.2.3 -requests==2.33.0 -spacy==3.7.5 +# Minimal runtime dependencies for local installs. +# +# Published package metadata in setup.py is the source of truth. Optional +# feature dependencies live behind package extras such as nlp, ocr, web, and all. pydantic==2.11.4 -Pillow==12.2.0 -sentencepiece==0.2.1 -protobuf==6.33.5 -pytesseract==0.3.13 -aiohttp==3.13.4 -numpy==1.26.4 -fastapi==0.115.12 -asyncio==3.4.3 -setuptools>=68.0.0 pydantic-settings==2.3.4 -typer==0.12.3 -sphinx==7.2.6 -cryptography==46.0.7 - -# Testing dependencies -pytest==9.0.3 -pytest-asyncio==0.21.0 -pytest-cov==4.1.0 +typing-extensions>=4.0 diff --git a/setup.py b/setup.py index 75284180..43eaff3b 100644 --- a/setup.py +++ b/setup.py @@ -24,64 +24,80 @@ ] # Optional heavy dependencies +nlp_deps = [ + "spacy>=3.7.0,<4.0", +] + +nlp_advanced_deps = [ + "gliner>=0.2.5", + "torch>=2.1.0,<2.7", + "transformers>=4.20.0", + "huggingface-hub>=0.16.0", +] + +ocr_deps = [ + "pytesseract>=0.3.0", + "Pillow>=12.2.0", + "sentencepiece>=0.2.0", + "protobuf>=4.0.0", +] + +distributed_deps = [ + "pandas>=2.0.0", + "numpy>=1.24.0", +] + +web_deps = [ + "fastapi>=0.100.0", + "aiohttp>=3.13.4", + "requests>=2.33.0", +] + +cli_deps = [ + "typer>=0.12.0", + "pydantic-settings>=2.0.0", +] + +crypto_deps = [ + "cryptography>=46.0.7", +] + +test_deps = [ + "pytest>=9.0.3", + "pytest-asyncio>=1.3.0", + "pytest-cov>=7.1.0", +] + +docs_deps = [ + "sphinx>=7.2.6", +] + +benchmark_deps = [ + "pytest-benchmark>=4.0.0", +] + extras_require = { - "nlp": [ - "spacy>=3.7.0,<4.0", - ], - "nlp-advanced": [ - "gliner>=0.2.5", - "torch>=2.1.0,<2.7", - "transformers>=4.20.0", - "huggingface-hub>=0.16.0", - ], - "ocr": [ - "pytesseract>=0.3.0", - "Pillow>=10.0.0", - "sentencepiece>=0.2.0", - "protobuf>=4.0.0", - ], - "distributed": [ - "pandas>=2.0.0", - "numpy>=1.24.0", - ], - "web": [ - "fastapi>=0.100.0", - "aiohttp>=3.8.0", - "requests>=2.30.0", - ], - "cli": [ - "typer>=0.12.0", - "pydantic-settings>=2.0.0", - ], - "crypto": [ - "cryptography>=40.0.0", - ], - "dev": [ - "pytest>=7.0.0", - "pytest-asyncio>=0.21.0", - "pytest-cov>=4.0.0", - "sphinx>=7.0.0", - ], + "nlp": nlp_deps, + "nlp-advanced": nlp_advanced_deps, + "ocr": ocr_deps, + "distributed": distributed_deps, + "web": web_deps, + "cli": cli_deps, + "crypto": crypto_deps, + "test": test_deps, + "docs": docs_deps, + "benchmark": benchmark_deps, + "dev": test_deps + docs_deps, # Convenience bundles - "all": [ - "spacy>=3.7.0,<4.0", - "gliner>=0.2.5", - "torch>=2.1.0,<2.7", - "transformers>=4.20.0", - "huggingface-hub>=0.16.0", - "pytesseract>=0.3.0", - "Pillow>=10.0.0", - "sentencepiece>=0.2.0", - "protobuf>=4.0.0", - "pandas>=2.0.0", - "numpy>=1.24.0", - "fastapi>=0.100.0", - "aiohttp>=3.8.0", - "requests>=2.30.0", - "typer>=0.12.0", - "pydantic-settings>=2.0.0", - "cryptography>=40.0.0", - ], + "all": ( + nlp_deps + + nlp_advanced_deps + + ocr_deps + + distributed_deps + + web_deps + + cli_deps + + crypto_deps + ), } setup( diff --git a/tox.ini b/tox.ini index 5e81c1f4..83d3f793 100644 --- a/tox.ini +++ b/tox.ini @@ -4,9 +4,11 @@ isolated_build = True [testenv:integration] deps = - -r requirements-dev.txt + -r requirements-test.txt pyspark>=3.0.0 -extras = all +extras = + all + test allowlist_externals = tesseract pip @@ -16,8 +18,10 @@ commands = [testenv] deps = - -r requirements-dev.txt -extras = all + -r requirements-test.txt +extras = + all + test allowlist_externals = tesseract pip @@ -29,9 +33,9 @@ commands = [testenv:lint] skip_install = true deps = - black - isort - flake8 + black==26.3.1 + isort==5.13.2 + flake8==7.0.0 commands = black --check . isort --check-only . @@ -39,7 +43,7 @@ commands = [testenv:typecheck] deps = - mypy + mypy==1.9.0 commands = mypy datafog tests