From 0cc37593d3e457051104e4e1ff931df5934b33ea Mon Sep 17 00:00:00 2001 From: Shiwani Mishra Date: Wed, 8 Apr 2026 21:45:20 +0530 Subject: [PATCH] fix: pin dependencies to exact versions using pip-tools Closes #868 Introduces pip-tools for reproducible dependency management. requirements.in is the human-maintained list of direct dependencies, deduplicated and normalised from the previous requirements.txt. requirements.txt is the pip-compiled lockfile with all transitive dependencies pinned to exact versions, generated with Python 3.12.3 to match CI. Adds update-deps Makefile target for regenerating the lockfile and documents the workflow in docs/developmentSetup.md. No changes to CI or install workflow: pip install -r requirements.txt continues to work as before. --- Makefile | 5 +- docs/developmentSetup.md | 17 + requirements.in | 107 ++++++ requirements.txt | 750 +++++++++++++++++++++++++++++++++------ 4 files changed, 763 insertions(+), 116 deletions(-) create mode 100644 requirements.in diff --git a/Makefile b/Makefile index fa11162e7..e93b97d88 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .ONESHELL: -.PHONY: run test covers install-deps dev docker lint frontend clean all +.PHONY: run test covers install-deps update-deps dev docker lint frontend clean all prod-run: gunicorn cre:app --log-file=- @@ -60,6 +60,9 @@ test: cover: . ./venv/bin/activate && FLASK_APP=cre.py FLASK_CONFIG=testing flask test --cover +update-deps: + pip install pip-tools && pip-compile requirements.in --output-file requirements.txt --strip-extras + install-deps-python: [ -d "./venv" ] && . ./venv/bin/activate &&\ pip install --upgrade pip setuptools &&\ diff --git a/docs/developmentSetup.md b/docs/developmentSetup.md index 501f8f322..ba511b18b 100644 --- a/docs/developmentSetup.md +++ b/docs/developmentSetup.md @@ -102,6 +102,23 @@ To edit files it is suggested that a code editor such as Visual Studio Code is u OpenCRE depends on Makefiles to automate the setup and execution of several aspects. You can install the project by running `make install`. +### Adding or updating Python dependencies + +Python dependencies are managed using [pip-tools](https://pip-tools.readthedocs.io/). + +- `requirements.in` — the human-maintained list of direct dependencies. **Edit this file** when adding or removing a package. +- `requirements.txt` — the auto-generated lockfile with all dependencies pinned to exact versions. **Do not edit this file directly.** + +After editing `requirements.in`, regenerate the lockfile with: + +```bash +make update-deps +``` + +Then commit both `requirements.in` and `requirements.txt`. + +> **Note:** Run `make update-deps` using the same Python version as CI (Python 3.12.3) to ensure the lockfile is consistent with the test environment. + ### Testing that everything works If the tests pass, the project should be operational. You can run tests with diff --git a/requirements.in b/requirements.in new file mode 100644 index 000000000..aeb7ce1f4 --- /dev/null +++ b/requirements.in @@ -0,0 +1,107 @@ +black==24.4.2 # do not touch, same version as superlinter +click +compliance-trestle +coverage +dacite +Flask +Flask_Caching +flask_compress +Flask_Cors +Flask_Migrate +Flask-SQLAlchemy +setuptools +gitpython +google +google-api-core +google-auth-oauthlib +google-cloud-aiplatform +google-cloud-trace +grpcio-status +grpcio +gspread +gunicorn +networkx +nltk +oauthlib +openai +playwright +psycopg2-binary +PyGithub +python-markdown-maker +scikit-learn +scipy +semver +SQLAlchemy +nose +mypy +numpy +neo4j +neomodel +openapi-schema-validator +openapi-spec-validator +openpyxl +orderedmultidict +packaging +paramiko +pathable +pathspec +pbr +pep517 +Pillow +pip-autoremove +platformdirs +pluggy +posthog +prance +prompt-toolkit +proto-plus +protobuf +pyasn1 +pyasn1-modules +pycodestyle +pycparser +pydantic +pyee +pyflakes +PyJWT +PyNaCl +pyparsing +pyrsistent +PySnooper +pytest +pytest-base-url +pytest-playwright +python-dateutil +python-dotenv +python-frontmatter +python-slugify +PyYAML==6.0.1 +regex +requests +requests-oauthlib +rfc3986 +rsa +rq +redis +ruamel.yaml +ruamel.yaml.clib +Shapely +six +smmap +sniffio +soupsieve +sqlalchemy-stubs +testresources +text-unidecode +threadpoolctl +toml +tomli +tqdm +types-PyYAML +typing-inspect +typing_extensions +untangle +urllib3 +vertexai +xmltodict +alive-progress diff --git a/requirements.txt b/requirements.txt index 573e7c00f..4c6995276 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,116 +1,636 @@ -black==24.4.2 # do not touch, same version as superlinter -click -compliance-trestle -coverage -dacite -Flask -Flask_Caching -flask_compress -Flask_Cors -Flask_Migrate -Flask-SQLAlchemy -setuptools -gitpython -google -google-api-core -google_auth_oauthlib -google-cloud-aiplatform -grpcio_status -grpcio -gspread -gunicorn -networkx -nltk -oauthlib -openai -playwright -psycopg2-binary -pygithub -python_markdown_maker -scikit_learn -scipy -semver -setuptools -SQLAlchemy -compliance-trestle -nose -mypy -numpy -neo4j -neomodel -openapi-schema-validator -openapi-spec-validator -openpyxl -orderedmultidict -packaging -paramiko -pathable -pathspec -pbr -pep517 -Pillow -pip-autoremove -platformdirs -playwright -pluggy -posthog -prance -prompt-toolkit -proto-plus -protobuf -psycopg2-binary -pyasn1 -pyasn1-modules -pycodestyle -pycparser -pydantic -pyee -pyflakes -PyGithub -PyJWT -PyNaCl -pyparsing -pyrsistent -PySnooper -pytest -pytest-base-url -pytest-playwright -python-dateutil -python-dotenv -python-frontmatter -python-markdown-maker -python-slugify -PyYAML==6.0.1 -regex -requests -requests-oauthlib -rfc3986 -rsa -rq -redis -ruamel.yaml -ruamel.yaml.clib -scikit-learn -Shapely -six -smmap -sniffio -soupsieve -SQLAlchemy -sqlalchemy-stubs -testresources -text-unidecode -threadpoolctl -toml -tomli -tqdm -types-PyYAML -typing-inspect -typing_extensions -untangle -urllib3 -vertexai -xmltodict -google-cloud-trace -alive-progress +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --output-file=requirements.txt --strip-extras requirements.in +# +about-time==4.2.1 + # via alive-progress +alembic==1.18.4 + # via flask-migrate +alive-progress==3.3.0 + # via -r requirements.in +annotated-types==0.7.0 + # via pydantic +anyio==4.13.0 + # via + # google-genai + # httpx + # openai +attrs==26.1.0 + # via + # compliance-trestle + # jsonschema + # referencing +backoff==2.2.1 + # via posthog +backports-zstd==1.3.0 + # via flask-compress +bcrypt==5.0.0 + # via paramiko +beautifulsoup4==4.14.3 + # via google +black==24.4.2 + # via -r requirements.in +blinker==1.9.0 + # via flask +brotli==1.2.0 + # via flask-compress +cachelib==0.13.0 + # via flask-caching +certifi==2026.2.25 + # via + # httpcore + # httpx + # requests +cffi==2.0.0 + # via + # cmarkgfm + # cryptography + # pynacl +chardet==7.4.1 + # via prance +charset-normalizer==3.4.7 + # via requests +click==8.3.2 + # via + # -r requirements.in + # black + # flask + # nltk + # rq +cmarkgfm==2025.10.22 + # via compliance-trestle +compliance-trestle==4.0.1 + # via -r requirements.in +coverage==7.13.5 + # via -r requirements.in +croniter==6.2.2 + # via rq +cryptography==46.0.5 + # via + # compliance-trestle + # google-auth + # paramiko + # pyjwt +dacite==1.9.2 + # via -r requirements.in +defusedxml==0.7.1 + # via + # compliance-trestle + # untangle +distro==1.9.0 + # via + # google-genai + # openai + # posthog +dnspython==2.8.0 + # via email-validator +docstring-parser==0.17.0 + # via google-cloud-aiplatform +email-validator==2.3.0 + # via pydantic +et-xmlfile==2.0.0 + # via openpyxl +flask==3.1.3 + # via + # -r requirements.in + # flask-caching + # flask-compress + # flask-cors + # flask-migrate + # flask-sqlalchemy +flask-caching==2.3.1 + # via -r requirements.in +flask-compress==1.24 + # via -r requirements.in +flask-cors==6.0.2 + # via -r requirements.in +flask-migrate==4.1.0 + # via -r requirements.in +flask-sqlalchemy==3.1.1 + # via + # -r requirements.in + # flask-migrate +furl==2.1.4 + # via compliance-trestle +gitdb==4.0.12 + # via gitpython +gitpython==3.1.46 + # via -r requirements.in +google==3.0.0 + # via -r requirements.in +google-api-core==2.30.2 + # via + # -r requirements.in + # google-cloud-aiplatform + # google-cloud-bigquery + # google-cloud-core + # google-cloud-resource-manager + # google-cloud-storage + # google-cloud-trace +google-auth==2.49.1 + # via + # google-api-core + # google-auth-oauthlib + # google-cloud-aiplatform + # google-cloud-bigquery + # google-cloud-core + # google-cloud-resource-manager + # google-cloud-storage + # google-cloud-trace + # google-genai + # gspread +google-auth-oauthlib==1.3.1 + # via + # -r requirements.in + # gspread +google-cloud-aiplatform==1.146.0 + # via + # -r requirements.in + # vertexai +google-cloud-bigquery==3.41.0 + # via google-cloud-aiplatform +google-cloud-core==2.5.1 + # via + # google-cloud-bigquery + # google-cloud-storage +google-cloud-resource-manager==1.17.0 + # via google-cloud-aiplatform +google-cloud-storage==3.10.1 + # via google-cloud-aiplatform +google-cloud-trace==1.19.0 + # via -r requirements.in +google-crc32c==1.8.0 + # via + # google-cloud-storage + # google-resumable-media +google-genai==1.70.0 + # via google-cloud-aiplatform +google-resumable-media==2.8.2 + # via + # google-cloud-bigquery + # google-cloud-storage +googleapis-common-protos==1.74.0 + # via + # google-api-core + # grpc-google-iam-v1 + # grpcio-status +graphemeu==0.7.2 + # via alive-progress +greenlet==3.3.2 + # via + # playwright + # sqlalchemy +grpc-google-iam-v1==0.14.4 + # via google-cloud-resource-manager +grpcio==1.80.0 + # via + # -r requirements.in + # google-api-core + # google-cloud-resource-manager + # google-cloud-trace + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-status +grpcio-status==1.80.0 + # via + # -r requirements.in + # google-api-core +gspread==6.2.1 + # via -r requirements.in +gunicorn==25.3.0 + # via -r requirements.in +h11==0.16.0 + # via httpcore +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via + # google-genai + # openai +idna==3.11 + # via + # anyio + # email-validator + # httpx + # requests +ilcli==0.3.2 + # via compliance-trestle +importlib-resources==6.5.2 + # via compliance-trestle +iniconfig==2.3.0 + # via pytest +invoke==3.0.3 + # via paramiko +itsdangerous==2.2.0 + # via flask +jinja2==3.1.6 + # via + # compliance-trestle + # flask +jiter==0.13.0 + # via openai +joblib==1.5.3 + # via + # nltk + # scikit-learn +jsonschema==4.24.1 + # via + # openapi-schema-validator + # openapi-spec-validator +jsonschema-path==0.4.5 + # via openapi-spec-validator +jsonschema-specifications==2025.9.1 + # via + # jsonschema + # openapi-schema-validator +lazy-object-proxy==1.12.0 + # via openapi-spec-validator +librt==0.8.1 + # via mypy +mako==1.3.10 + # via alembic +markupsafe==3.0.3 + # via + # flask + # jinja2 + # mako + # werkzeug +mypy==1.20.0 + # via + # -r requirements.in + # sqlalchemy-stubs +mypy-extensions==1.1.0 + # via + # black + # mypy + # typing-inspect +neo4j==6.1.0 + # via + # -r requirements.in + # neomodel +neomodel==6.1.0 + # via -r requirements.in +networkx==3.4.2 + # via -r requirements.in +nltk==3.9.4 + # via -r requirements.in +nose==1.3.7 + # via -r requirements.in +numpy==2.2.6 + # via + # -r requirements.in + # scikit-learn + # scipy + # shapely +oauthlib==3.3.1 + # via + # -r requirements.in + # requests-oauthlib +openai==2.30.0 + # via -r requirements.in +openapi-schema-validator==0.8.1 + # via + # -r requirements.in + # openapi-spec-validator +openapi-spec-validator==0.8.4 + # via -r requirements.in +openpyxl==3.1.5 + # via + # -r requirements.in + # compliance-trestle +orderedmultidict==1.0.2 + # via + # -r requirements.in + # furl +orjson==3.11.8 + # via compliance-trestle +packaging==26.0 + # via + # -r requirements.in + # black + # google-cloud-aiplatform + # google-cloud-bigquery + # gunicorn + # prance + # pytest +paramiko==4.0.0 + # via + # -r requirements.in + # compliance-trestle +pathable==0.5.0 + # via + # -r requirements.in + # jsonschema-path +pathspec==1.0.4 + # via + # -r requirements.in + # black + # mypy +pbr==7.0.3 + # via + # -r requirements.in + # testresources +pep517==0.13.1 + # via -r requirements.in +pillow==12.2.0 + # via -r requirements.in +pip-autoremove==0.10.0 + # via -r requirements.in +platformdirs==4.9.4 + # via + # -r requirements.in + # black +playwright==1.58.0 + # via + # -r requirements.in + # pytest-playwright +pluggy==1.6.0 + # via + # -r requirements.in + # pytest +posthog==7.10.2 + # via -r requirements.in +prance==25.4.8.0 + # via -r requirements.in +prompt-toolkit==3.0.52 + # via -r requirements.in +proto-plus==1.27.2 + # via + # -r requirements.in + # google-api-core + # google-cloud-aiplatform + # google-cloud-resource-manager + # google-cloud-trace +protobuf==6.33.6 + # via + # -r requirements.in + # google-api-core + # google-cloud-aiplatform + # google-cloud-resource-manager + # google-cloud-trace + # googleapis-common-protos + # grpc-google-iam-v1 + # grpcio-status + # proto-plus +psycopg2-binary==2.9.11 + # via -r requirements.in +pyasn1==0.6.3 + # via + # -r requirements.in + # pyasn1-modules + # rsa +pyasn1-modules==0.4.2 + # via + # -r requirements.in + # google-auth +pycodestyle==2.14.0 + # via -r requirements.in +pycparser==3.0 + # via + # -r requirements.in + # cffi +pydantic==2.12.5 + # via + # -r requirements.in + # compliance-trestle + # google-cloud-aiplatform + # google-genai + # openai + # openapi-schema-validator + # openapi-spec-validator + # pydantic-settings +pydantic-core==2.41.5 + # via pydantic +pydantic-settings==2.13.1 + # via + # openapi-schema-validator + # openapi-spec-validator +pyee==13.0.1 + # via + # -r requirements.in + # playwright +pyflakes==3.4.0 + # via -r requirements.in +pygithub==2.9.0 + # via -r requirements.in +pygments==2.20.0 + # via pytest +pyjwt==2.12.1 + # via + # -r requirements.in + # pygithub +pynacl==1.6.2 + # via + # -r requirements.in + # paramiko + # pygithub +pyparsing==3.3.2 + # via -r requirements.in +pyrsistent==0.20.0 + # via -r requirements.in +pysnooper==1.2.3 + # via -r requirements.in +pytest==9.0.3 + # via + # -r requirements.in + # pytest-base-url + # pytest-playwright +pytest-base-url==2.1.0 + # via + # -r requirements.in + # pytest-playwright +pytest-playwright==0.7.2 + # via -r requirements.in +python-dateutil==2.9.0.post0 + # via + # -r requirements.in + # croniter + # google-cloud-bigquery + # posthog +python-dotenv==1.2.2 + # via + # -r requirements.in + # compliance-trestle + # pydantic-settings +python-frontmatter==1.1.0 + # via + # -r requirements.in + # compliance-trestle +python-markdown-maker==1.0 + # via -r requirements.in +python-slugify==8.0.4 + # via + # -r requirements.in + # pytest-playwright +pytz==2026.1.post1 + # via neo4j +pyyaml==6.0.1 + # via + # -r requirements.in + # jsonschema-path + # python-frontmatter +redis==7.4.0 + # via + # -r requirements.in + # rq +referencing==0.37.0 + # via + # jsonschema + # jsonschema-path + # jsonschema-specifications + # openapi-schema-validator +regex==2026.4.4 + # via + # -r requirements.in + # nltk +requests==2.33.1 + # via + # -r requirements.in + # compliance-trestle + # google-api-core + # google-auth + # google-cloud-bigquery + # google-cloud-storage + # google-genai + # posthog + # prance + # pygithub + # pytest-base-url + # requests-oauthlib +requests-oauthlib==2.0.0 + # via + # -r requirements.in + # google-auth-oauthlib +rfc3339-validator==0.1.4 + # via openapi-schema-validator +rfc3986==2.0.0 + # via -r requirements.in +rpds-py==0.30.0 + # via + # jsonschema + # referencing +rq==2.7.0 + # via -r requirements.in +rsa==4.9.1 + # via -r requirements.in +ruamel-yaml==0.19.1 + # via + # -r requirements.in + # compliance-trestle + # prance +ruamel-yaml-clib==0.2.15 + # via -r requirements.in +scikit-learn==1.7.2 + # via -r requirements.in +scipy==1.15.3 + # via + # -r requirements.in + # scikit-learn +semver==3.0.4 + # via -r requirements.in +shapely==2.1.2 + # via -r requirements.in +six==1.17.0 + # via + # -r requirements.in + # furl + # orderedmultidict + # posthog + # python-dateutil + # rfc3339-validator +smmap==5.0.3 + # via + # -r requirements.in + # gitdb +sniffio==1.3.1 + # via + # -r requirements.in + # google-genai + # openai +soupsieve==2.8.3 + # via + # -r requirements.in + # beautifulsoup4 +sqlalchemy==2.0.49 + # via + # -r requirements.in + # alembic + # flask-sqlalchemy +sqlalchemy-stubs==0.4 + # via -r requirements.in +tenacity==9.1.4 + # via google-genai +testresources==2.0.2 + # via -r requirements.in +text-unidecode==1.3 + # via + # -r requirements.in + # python-slugify +threadpoolctl==3.6.0 + # via + # -r requirements.in + # scikit-learn +toml==0.10.2 + # via -r requirements.in +tomli==2.4.1 + # via -r requirements.in +tqdm==4.67.3 + # via + # -r requirements.in + # nltk + # openai +types-pyyaml==6.0.12.20260408 + # via -r requirements.in +typing-extensions==4.15.0 + # via + # -r requirements.in + # alembic + # anyio + # beautifulsoup4 + # google-cloud-aiplatform + # google-genai + # grpcio + # mypy + # neomodel + # openai + # posthog + # pydantic + # pydantic-core + # pyee + # pygithub + # referencing + # sqlalchemy + # sqlalchemy-stubs + # typing-inspect + # typing-inspection +typing-inspect==0.9.0 + # via -r requirements.in +typing-inspection==0.4.2 + # via + # pydantic + # pydantic-settings +untangle==1.2.1 + # via -r requirements.in +urllib3==2.6.3 + # via + # -r requirements.in + # pygithub + # requests +vertexai==1.43.0 + # via -r requirements.in +wcwidth==0.6.0 + # via prompt-toolkit +websockets==16.0 + # via google-genai +werkzeug==3.1.8 + # via + # flask + # flask-cors +xmltodict==1.0.4 + # via -r requirements.in +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools