diff --git a/.gitignore b/.gitignore index 65fc867..e15106e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,22 +1,216 @@ -*~ +# Byte-compiled / optimized / DLL files __pycache__/ -/build/ -/docs/_build/ -/docs/generated/ -.DS_Store -/.vscode/ -/*.egg-info/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg *.egg -*.prof -*.coverage -*.pickle -/out/ +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: *.log -/dist/ -*.pye -*.pyc -*.kate-swp -/.tox/ -/.idea/ -/.pytest_cache/ -/.cache/ +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +# Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +# poetry.lock +# poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +# pdm.lock +# pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +# pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# Redis +*.rdb +*.aof +*.pid + +# RabbitMQ +mnesia/ +rabbitmq/ +rabbitmq-data/ + +# ActiveMQ +activemq-data/ + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +# .idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +# .vscode/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +# Streamlit +.streamlit/secrets.toml diff --git a/README.md b/README.md index d3b3d03..5a4ae0a 100644 --- a/README.md +++ b/README.md @@ -6,4 +6,189 @@ [badge-ci]: https://img.shields.io/github/actions/workflow/status/saezlab/cache-manager/ci.yml?branch=main [link-ci]: https://github.com/saezlab/cache-manager/actions/workflows/ci.yml -# A Cache Manager Python Module +# cache-manager + +## Description +cache-manager is a lightweight, Pythonic cache for files with an SQLite registry. It lets you: + +- Store files under a cache directory while tracking metadata in SQLite. +- Version entries automatically based on a stable key derived from a URI/parameters. +- Track and query item status (UNINITIALIZED, WRITE, READY, FAILED, TRASH). +- Attach type-aware attributes (int, float, varchar, datetime, text) and search by them. +- Open plain and compressed files (gz, zip, tar(.gz|.bz2)) via a unified interface. +- Clean up stale records/files and keep the best/most recent ready entries. + +This is ideal for reproducible data pipelines and ETL steps where you want deterministic, discoverable artifacts. + +## Table of Contents +- [cache-manager](#cache-manager) + - [Description](#description) + - [Table of Contents](#table-of-contents) + - [Installation](#installation) + - [Usage](#usage) + - [Configuration](#configuration) + - [Examples](#examples) + - [Contributing](#contributing) + - [License](#license) + - [Contact](#contact) + +## Installation +Clone and install in editable mode (no extra tools required): + +```bash +git clone https://github.com/saezlab/cache-manager.git +cd cache-manager +python -m venv .venv +source .venv/bin/activate +pip install -e . +``` + +Alternatively, if you prefer Poetry: + +```bash +git clone https://github.com/saezlab/cache-manager.git +cd cache-manager +poetry install +``` + +## Usage +The API centers around two types: `Cache` (manager) and `CacheItem` (one file + metadata). Create a cache, create or retrieve items, write files, mark them READY, and open them later. + +Minimal example: + +```python +import cache_manager as cm +from cache_manager._status import Status + +cache = cm.Cache(path="./my_cache") +item = cache.create( + uri="https://example.org/data.tsv", + params={"dataset": "demo", "version": 1}, + attrs={"species": "human", "rows": 1200}, + status=Status.WRITE.value, + filename="data.tsv", +) + +with open(item.path, "w", encoding="utf-8") as f: + f.write("col_a\tcol_b\n1\t2\n") + +item.ready() +best = cache.best( + uri="https://example.org/data.tsv", + params={"dataset": "demo", "version": 1}, +) +print(best, best.path) +``` + +Run the included example script which downloads a real dataset and caches it: + +```bash +python scripts/hello_cache_manager.py +``` + +## Configuration +There is no global config file; you configure the cache per instance: + +- `Cache(path: str | None = None, pkg: str | None = None)` + - `path`: explicit directory for cache (contains the SQLite registry and files). + - `pkg`: if set, uses an OS-specific cache directory for that application name via `platformdirs` (e.g., on Linux: `~/.cache/`). + +Common item fields: +- `uri` (str): a canonical identifier used for the hash key (together with `params`). +- `params` (dict): serialized to the stable key; changing them yields a new key. +- `attrs` (dict): typed attributes persisted to attribute tables for rich queries. +- `status` (int): from `cache_manager._status.Status` (READY, WRITE, etc.). +- `filename` (str): filename to be used in the cache; extension is auto-inferred. + +Logging/session helpers are available under `cache_manager.session` and `cache_manager.log` if you want simple trace output. + +## Examples + +1. Create or reuse an item with `best_or_new`. + +```python +import os +import cache_manager as cm +from cache_manager._status import Status + +cache = cm.Cache(path="./my_cache") +uri = "https://example.org/report.csv" +params = {"year": 2026, "cohort": "A"} + +item = cache.best_or_new( + uri=uri, + params=params, + attrs={"kind": "report", "format": "csv"}, + filename="report.csv", + new_status=Status.WRITE.value, +) + +if item.status != Status.READY.value or not os.path.exists(item.path): + with open(item.path, "w", encoding="utf-8") as f: + f.write("id,value\n1,42\n") + item.ready() + +print("Using:", item.path) +``` + +2. Query by attributes and metadata. + +```python +import cache_manager as cm +from cache_manager._status import Status + +cache = cm.Cache(path="./my_cache") + +cache.create( + uri="demo://sample-1", + attrs={"project": "alpha", "batch": 1, "score": 0.95}, + status=Status.READY.value, +) +cache.create( + uri="demo://sample-2", + attrs={"project": "alpha", "batch": 2, "score": 0.71}, + status=Status.READY.value, +) + +ids = cache.by_attrs({"project": "alpha", "batch": 2}) +print("matching ids:", ids) + +items = cache.search(uri="demo://sample-2", status=Status.READY.value) +for it in items: + print(it.version_id, it.attrs) +``` + +3. Open a cached file through `CacheItem.open`. + +```python +import cache_manager as cm +from cache_manager._status import Status + +cache = cm.Cache(path="./my_cache") +item = cache.create( + uri="demo://text-file", + filename="hello.txt", + status=Status.WRITE.value, +) + +with open(item.path, "w", encoding="utf-8") as f: + f.write("hello\nworld\n") + +item.ready() + +opened = item.open(default_mode="r", encoding="utf-8", large=True) +print(next(iter(opened.result)).strip()) +``` + +## Contributing +Contributions are welcome! A typical flow: + +Please open issues and pull requests on GitHub. If you plan a larger change, consider discussing it in an issue first. (A dedicated `CONTRIBUTING.md` may be added later.) + +## License +This project is licensed under the GNU General Public License v3.0. See the [LICENSE](LICENSE) file for details. + +## Contact +OmniPath Team - omnipathdb@gmail.com + +Project page: https://github.com/saezlab/cache-manager diff --git a/scripts/hello_cache_manager.py b/scripts/hello_cache_manager.py new file mode 100644 index 0000000..792b31a --- /dev/null +++ b/scripts/hello_cache_manager.py @@ -0,0 +1,101 @@ +import os +import csv +from urllib.request import urlopen +from contextlib import closing + +import cache_manager as cm +from cache_manager._status import Status + + +URL = ( + "https://zenodo.org/records/16902349/files/" + "synthetic_protein_interactions.tsv?download=1" +) +FILENAME = "synthetic_protein_interactions.tsv" + + +def download_to(path: str, url: str, chunk_size: int = 1024 * 1024) -> None: + """Download URL to `path` using stdlib only (streamed).""" + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + with closing(urlopen(url, timeout=120)) as resp, open(path, "wb") as out: + while True: + chunk = resp.read(chunk_size) + if not chunk: + break + out.write(chunk) + + +def main(): + # 1) Create / open cache in a local directory + cache = cm.Cache(path="./my_cache") + + # 2) Get best READY item for this URI or create a new one with WRITE status + params = { + "source": "Zenodo", + "record": 16902349, + "file": FILENAME, + } + attrs = { + "project": "demo", + "dataset": "synthetic_protein_interactions", + "format": "tsv", + } + + item = cache.best_or_new( + uri=URL, + params=params, + attrs=attrs, + filename=FILENAME, + new_status=Status.WRITE.value, + ) + + print("Item:", item) + print("Cache dir:", cache.dir) + print("Target path:", item.path) + + # 3) Download if not READY or file missing + need_download = item.status != Status.READY.value or not os.path.exists(item.path) + if need_download: + try: + print("Downloading from:", URL) + download_to(item.path, URL) + item.ready() + print("Download complete. Marked READY.") + except Exception as e: + item.failed() + # Best effort cleanup of partial download + try: + if os.path.exists(item.path): + os.remove(item.path) + except Exception: + pass + raise RuntimeError(f"Download failed: {e}") + else: + print("Already READY in cache; skipping download.") + + # 4) Open and read a few lines (header + 5 rows) + fobj = item.open(mode="r", encoding="utf-8") + try: + reader = csv.reader(fobj, delimiter="\t") + rows = [] + for i, row in enumerate(reader): + rows.append(row) + if i >= 5: # header + 5 + break + print("Preview (first 6 lines):") + for r in rows: + print(r) + finally: + if hasattr(fobj, "close"): + fobj.close() + + # 5) Show how to find this item again (best READY) + best_ready = cache.best(uri=URL) # defaults to status=READY + print("Best READY version:", best_ready) + + # 6) Attribute-based search + print("By attrs (project=demo):", cache.by_attrs({"project": "demo"})) + + +if __name__ == "__main__": + main() \ No newline at end of file