From a2457f398aa0634491b3836e7f7e8c80bac70249 Mon Sep 17 00:00:00 2001 From: han-hoff Date: Wed, 1 Jul 2026 10:45:36 -0300 Subject: [PATCH 1/3] harden untrusted-input, offline embedders, model_hash gate, CI & provenance Remediates the high/medium findings from the security/privacy/compliance audit. Each fix ships with a test; full Rust + Python suites green. security - HNSW decode: range-check neighbour ids, entry_point and max_level against n_nodes so a hostile .nest can no longer index out of bounds (OOB panic/DoS) on any search path (ann/codec.rs). - zstd sections: cap decompression at max(64 MiB, 128x compressed) and reject frames declaring more, killing the decompression-bomb OOM at open (zstd_codec.rs). - BM25 v1/v2 decode: bound with_capacity hints from attacker-controlled counts (n_docs/n_terms/df), killing the allocation-bomb abort (bm25/codec.rs). - model_hash honesty gate on the python surface: NestFile exposes model_hash, retrieve accepts expected_model_hash and raises on mismatch, and the flagship forge/retrieve.py verifies by default (was CLI-only; the advertised binding silently lost it). - AVX2 int8 kernel: replace the unaligned *const i64 deref with read_unaligned (was UB on the exact-rerank hot path). privacy - force HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE by default in every sentence-transformers entry point (embed_query, model_fingerprint, nest_build_corpus, convert_legacy); network is opt-in via NEST_ALLOW_DOWNLOAD. Stops an unexpected model fetch mid-run while handling data. - resolve the HF snapshot the loaded revision points to (refs/main), not the alphabetical sorted()[0], so the model fingerprint attests the real model. - EmbeddingCache is keyed by (chunk_id, model): re-embedding a corpus with a different model no longer reuses stale vectors under a new model_hash. compliance / governance - add CI (.github/workflows/ci.yml): build/test/clippy/fmt --locked + python tests + cargo-audit, so the gate is not laptop-only. - track Cargo.lock (pin the rust dep set for reproducibility + CVE triage). - add --version to the CLI; correct SECURITY.md (supported versions, unsafe scope, release provenance) and the README tier-1 citation wording. - doc/data-governance.md: erasure/rotation posture for an immutable distributed .nest, provenance gaps, and encryption-at-rest requirement for sensitive data. - dat/demo/README.md: per-source corpus license bill of materials. --- .github/workflows/ci.yml | 72 ++ .gitignore | 3 +- Cargo.lock | 1018 +++++++++++++++++ README.md | 2 +- SECURITY.md | 12 +- crates/nest-cli/src/main.rs | 1 + crates/nest-format/src/encoding/zstd_codec.rs | 70 +- crates/nest-python/src/nest_file.rs | 10 +- crates/nest-python/src/retrieve_fn.rs | 18 + crates/nest-runtime/src/ann/codec.rs | 58 +- crates/nest-runtime/src/bm25/codec.rs | 54 +- crates/nest-runtime/src/mmap_file.rs | 11 + crates/nest-runtime/src/simd/avx2.rs | 10 +- dat/demo/README.md | 30 + doc/data-governance.md | 76 ++ python/builder.py | 36 +- python/convert_legacy.py | 7 + python/embed_query.py | 23 +- python/forge/retrieve.py | 14 +- python/forge/test_retrieve.py | 1 + python/model_fingerprint.py | 54 +- python/nest.py | 3 +- python/tools/nest_build_corpus.py | 9 + tests/test_builder.py | 77 +- tests/test_offline_guard.py | 53 + 25 files changed, 1666 insertions(+), 56 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 Cargo.lock create mode 100644 doc/data-governance.md create mode 100644 tests/test_offline_guard.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..c882c4ae --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,72 @@ +name: ci + +# Runs the quality gate on every push to main and every PR, so the checks that +# previously only ran on the maintainer's laptop (release_check.sh) produce +# independent, logged, reproducible evidence. Gate merges on this workflow. +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + rust: + name: rust · build · test · clippy · fmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy, rustfmt + - run: cargo fmt --all --check + - run: cargo build --workspace --locked + - run: cargo clippy --workspace --all-targets --locked -- -D warnings + - run: cargo test --workspace --locked + + python: + name: python · tests · ruff + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + lfs: true # hydrate the vendored potion table for the forge self-tests + - uses: dtolnay/rust-toolchain@stable + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install --upgrade numpy tokenizers ruff + - name: build extension + cli + run: | + cargo build --release -p nest-cli --locked + PYO3_PYTHON="$(which python)" cargo build --release -p nest-python --locked + cp target/release/lib_nest.so python/_nest.so + - name: engine + builder tests + run: | + python tests/test_e2e.py + python tests/test_builder.py + python tests/test_search_text_model_hash.py + python tests/test_offline_guard.py + - name: forge self-tests (offline, socket-blocked) + run: | + python python/forge/test_embed_potion.py + python python/forge/test_retrieve.py + - run: ruff check python tests + + audit: + name: cargo-audit (advisory) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - run: cargo install cargo-audit --locked + # Informational until the pre-existing advisory backlog is triaged: it + # surfaces vulnerable deps in the logs without blocking merges. Flip to a + # blocking check once the tree is clean. + - run: cargo audit + continue-on-error: true diff --git a/.gitignore b/.gitignore index 8f96d56b..8072595e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /target -/Cargo.lock +# Cargo.lock IS tracked: nest ships a CLI binary, so a committed lockfile is +# required for reproducible builds and CVE triage against a pinned dep set. **/*.rs.bk *.pdb *.so diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 00000000..a5f45f22 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1018 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cc" +version = "1.2.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core", + "wasip2", + "wasip3", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "bytemuck", + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "memchr" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" + +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + +[[package]] +name = "nest-cli" +version = "0.3.0" +dependencies = [ + "anyhow", + "clap", + "hex", + "memmap2", + "nest-format", + "nest-runtime", + "rand", + "serde", + "serde_json", +] + +[[package]] +name = "nest-format" +version = "0.3.0" +dependencies = [ + "half", + "hex", + "serde", + "serde_json", + "sha2", + "tempfile", + "thiserror", + "zstd", +] + +[[package]] +name = "nest-python" +version = "0.3.0" +dependencies = [ + "nest-format", + "nest-runtime", + "pyo3", + "pyo3-build-config", + "serde_json", +] + +[[package]] +name = "nest-runtime" +version = "0.3.0" +dependencies = [ + "half", + "hex", + "libc", + "memmap2", + "nest-format", + "rayon", + "serde", + "serde_json", + "sha2", + "thiserror", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "indexmap", + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/README.md b/README.md index 88f703b7..560cd121 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ four properties, all enforced by the format itself, not by policy. **self-contained.** a `.nest` file is the entire knowledge base. chunks, embeddings, source spans, search contract, indices. you copy it like you copy a SQLite db. no schema migrations, no companion files, no external service to look up. -**cryptographically verifiable.** every section has its own SHA-256 over the physical bytes. the file has a SHA-256 footer. the canonical content has a third hash, computed over the decoded bytes, that survives compression and lets two builds of the same logical content prove they are the same content. every hit comes back with a `nest://content_hash/chunk_id` citation that reopens the exact byte span it came from. +**cryptographically verifiable.** every section has its own SHA-256 over the physical bytes. the file has a SHA-256 footer. the canonical content has a third hash, computed over the decoded bytes, that survives compression and lets two builds of the same logical content prove they are the same content. every hit comes back with a `nest://content_hash/chunk_id` citation that resolves (via `nest cite`) to the stored canonical text of that chunk plus its recorded source offsets and verifying hashes — tier-1, not a reopen of the original source document (which the self-contained file does not carry). **reproducible.** same chunks plus same model fingerprint plus `reproducible=True` produce byte-identical files. two operators on two machines build the same `file_hash`. this is what makes the citation URI useful: it points at content, not at a copy. diff --git a/SECURITY.md b/SECURITY.md index 75538837..ac0246fc 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -8,7 +8,8 @@ only the latest minor on `main` is supported. | version | status | |---------|--------| -| 0.2.x | supported | +| 0.3.x | supported (current) | +| 0.2.x | not supported, please upgrade | | 0.1.x | not supported, please upgrade | ## reporting a vulnerability @@ -52,7 +53,8 @@ things we do not treat as security bugs: ## hardening notes -- the runtime never opens a network socket. queries are answered from `mmap`. -- `model_hash` is a granular fingerprint over the local model snapshot (config + tokenizer + weights + pooling + dim + normalize). a mismatch fails with a typed error, never silently. -- `unsafe` is concentrated in the SIMD dispatcher (`crates/nest-runtime/src/simd/`) and the mmap reader (`crates/nest-runtime/src/mmap_file.rs`). every `unsafe` block carries a `// SAFETY:` comment documenting the invariant. -- binary releases of `nest-cli` are not yet signed. signed tags are on the v0.3 backlog. +- the runtime (rust) never opens a network socket. queries are answered from `mmap`. the default query embedders are offline too: `ask`/`retrieve` use the vendored potion table (no network by construction), and the `search-text` sentence-transformers path forces `HF_HUB_OFFLINE`/`TRANSFORMERS_OFFLINE` unless you opt in with `NEST_ALLOW_DOWNLOAD=1` (or pass `--model-path`). +- `model_hash` is a granular fingerprint over the local model snapshot (config + tokenizer + weights + pooling + dim + normalize). a mismatch fails with a typed error, never silently. the CLI (`search-text`) enforces this; the Python `NestFile.retrieve` binding accepts `expected_model_hash` and the flagship `forge/retrieve.py` passes it by default, so the honesty gate holds on the Python surface too. +- `unsafe` lives in the SIMD dispatcher (`crates/nest-runtime/src/simd/`), the mmap reader (`crates/nest-runtime/src/mmap_file.rs`), and a handful of zero-copy view casts in `crates/nest-format/src/layout/` and `encoding/int8.rs`. the SIMD and mmap sites carry `// SAFETY:` comments; documenting the remaining `nest-format` sites is a tracked hardening item (do not assume every block is annotated). +- untrusted `.nest` files: the header/section/footer checksums are unkeyed SHA-256 (corruption detection, NOT authenticity) — an attacker can recompute them, so `validate()` does not prove a file is trustworthy. Safety against a hostile file rests on the parser's memory-safety (bounds-checked indices, capped decompression/allocation); opening an untrusted corpus still executes that parser, so treat unknown `.nest` files with the same care as any untrusted input. +- release provenance: commits are signed (ssh signing), but release tags and `nest-cli` binaries are NOT yet cryptographically signed, and no SBOM is published per release. Treat a downloaded artifact as unverified against source until signed releases land. `Cargo.lock` is committed so the rust dependency set is pinned and auditable. This is a tracked hardening item. diff --git a/crates/nest-cli/src/main.rs b/crates/nest-cli/src/main.rs index 9e8db3da..f57c282b 100644 --- a/crates/nest-cli/src/main.rs +++ b/crates/nest-cli/src/main.rs @@ -6,6 +6,7 @@ mod cmd; #[derive(Parser)] #[command(name = "nest")] +#[command(version)] #[command(about = ".nest — Semantic Knowledge Format for Local Agents", long_about = None)] struct Cli { #[command(subcommand)] diff --git a/crates/nest-format/src/encoding/zstd_codec.rs b/crates/nest-format/src/encoding/zstd_codec.rs index cc63e05f..97f523c9 100644 --- a/crates/nest-format/src/encoding/zstd_codec.rs +++ b/crates/nest-format/src/encoding/zstd_codec.rs @@ -16,10 +16,74 @@ pub fn zstd_encode(bytes: &[u8]) -> crate::Result> { .map_err(|e| NestError::InvalidInput(format!("zstd compression failed: {}", e))) } +/// hostile-input guard: an attacker can ship a tiny zstd frame that inflates +/// to many GiB and OOM-kills the process at file `open()` (a classic +/// decompression bomb). The decompressed size is bounded to +/// `max(MIN_DECOMPRESS_CAP, MAX_DECOMPRESS_RATIO * compressed_len)`: the floor +/// keeps small legitimate sections working, the ratio bounds amplification on +/// larger inputs. The floor matches the per-stream `STREAM_CAP` the dict codec +/// already uses (`zstd_dict.rs`). +const MIN_DECOMPRESS_CAP: usize = 64 * 1024 * 1024; +const MAX_DECOMPRESS_RATIO: usize = 128; + +fn decompress_cap(compressed_len: usize) -> usize { + compressed_len + .saturating_mul(MAX_DECOMPRESS_RATIO) + .max(MIN_DECOMPRESS_CAP) +} + /// lDecompress a zstd payload. Internal helper used by `decode_payload`. +/// Bounded by [`decompress_cap`] so a decompression bomb cannot exhaust memory +/// at open time: a frame that declares (or expands past) more than the cap is +/// rejected before the large allocation, never a panic. pub(super) fn zstd_decode(bytes: &[u8]) -> crate::Result> { - zstd::decode_all(bytes).map_err(|e| NestError::MalformedSectionPayload { + let cap = decompress_cap(bytes.len()); + let bomb = |reason: String| NestError::MalformedSectionPayload { section_id: 0, - reason: format!("zstd decompression failed: {}", e), - }) + reason, + }; + let fail = |e: std::io::Error| bomb(format!("zstd decompression failed: {}", e)); + match zstd::bulk::Decompressor::upper_bound(bytes) { + // declared frame size already exceeds the cap: refuse before allocating. + Some(declared) if declared > cap => Err(bomb(format!( + "zstd frame declares {} bytes, exceeds cap {}", + declared, cap + ))), + // declared size known and within cap: allocate exactly it. + Some(declared) => zstd::bulk::decompress(bytes, declared).map_err(fail), + // no size hint: allow up to the cap, erroring if the frame exceeds it. + None => zstd::bulk::decompress(bytes, cap).map_err(fail), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn roundtrip_within_cap() { + let original = b"the file is the database ".repeat(256); + let compressed = zstd_encode(&original).unwrap(); + let decoded = zstd_decode(&compressed).unwrap(); + assert_eq!(decoded, original); + } + + #[test] + fn decompression_bomb_is_rejected_not_ooming() { + // a tiny compressed frame declaring more than the cap must error, + // never inflate. zeros compress to a few bytes but declare their full + // size in the frame header, which `upper_bound` reads. + let bomb = vec![0u8; MIN_DECOMPRESS_CAP + 1]; + let compressed = zstd_encode(&bomb).unwrap(); + assert!(compressed.len() < 4096, "bomb should compress tiny"); + let err = zstd_decode(&compressed).unwrap_err(); + assert!(matches!(err, NestError::MalformedSectionPayload { .. })); + } + + #[test] + fn cap_has_floor_and_ratio() { + assert_eq!(decompress_cap(0), MIN_DECOMPRESS_CAP); + assert_eq!(decompress_cap(usize::MAX), usize::MAX); // saturating, no overflow + assert_eq!(decompress_cap(MIN_DECOMPRESS_CAP), MIN_DECOMPRESS_CAP * MAX_DECOMPRESS_RATIO); + } } diff --git a/crates/nest-python/src/nest_file.rs b/crates/nest-python/src/nest_file.rs index 516f0e8d..7f14734c 100644 --- a/crates/nest-python/src/nest_file.rs +++ b/crates/nest-python/src/nest_file.rs @@ -89,7 +89,7 @@ impl NestFile { /// tier-1 stored canonical `text`, the verifying hashes, the stable /// citation_id, and the rerank-source precision marker. embed the query /// OFFLINE first (see python/forge/retrieve.py for the potion path). - #[pyo3(signature = (query, k, candidates=None, hops=1, ef=100))] + #[pyo3(signature = (query, k, candidates=None, hops=1, ef=100, expected_model_hash=None))] fn retrieve( &self, query: &Bound, @@ -97,8 +97,9 @@ impl NestFile { candidates: Option, hops: usize, ef: usize, + expected_model_hash: Option, ) -> PyResult> { - crate::retrieve_fn::retrieve(&self.rt, query, k, candidates, hops, ef) + crate::retrieve_fn::retrieve(&self.rt, query, k, candidates, hops, ef, expected_model_hash) } #[getter] @@ -136,6 +137,11 @@ impl NestFile { self.rt.has_graph() } + #[getter] + fn model_hash(&self) -> String { + self.rt.model_hash().to_string() + } + #[getter] fn file_hash(&self) -> String { self.rt.file_hash().to_string() diff --git a/crates/nest-python/src/retrieve_fn.rs b/crates/nest-python/src/retrieve_fn.rs index 00cb1cc2..63283c25 100644 --- a/crates/nest-python/src/retrieve_fn.rs +++ b/crates/nest-python/src/retrieve_fn.rs @@ -63,7 +63,25 @@ pub fn retrieve( candidates: Option, hops: usize, ef: usize, + expected_model_hash: Option, ) -> PyResult> { + // lhonesty gate: when the caller passes the model_hash of the embedder it + // used for `query`, reject a corpus built with a different model. A bare + // query vector carries no model identity, so the runtime cannot gate + // unconditionally the way the CLI does — the caller opts in by passing the + // hash (forge/retrieve.py does so by default). Without it, behaviour is + // unchanged, but a mismatch would silently return cosine-valid, wrong hits. + if let Some(expected) = &expected_model_hash { + let actual = rt.model_hash(); + if expected != actual { + return Err(PyValueError::new_err(format!( + "model_hash mismatch: the query was embedded with {expected}, but the corpus \ + was built with {actual}. Results would be cosine-valid but semantically wrong. \ + Pass expected_model_hash=None to bypass this check." + ))); + } + } + let qvec: Vec = query .extract() .map_err(|e| PyValueError::new_err(format!("invalid query vector: {e}")))?; diff --git a/crates/nest-runtime/src/ann/codec.rs b/crates/nest-runtime/src/ann/codec.rs index b708ab89..49f1bf08 100644 --- a/crates/nest-runtime/src/ann/codec.rs +++ b/crates/nest-runtime/src/ann/codec.rs @@ -79,6 +79,17 @@ impl HnswIndex { })); } }; + // hostile-input guard: `entry_point` and `max_level` are parsed from + // untrusted bytes and later index `nodes` / drive the layer descent at + // search time. Neighbour ids are range-checked in decode_nodes_v{1,2}; + // validate the two scalar fields here so a crafted payload can never + // index out of range (an OOB slice = guaranteed panic / DoS). + if max_level as u64 > MAX_LEVEL { + return Err(malformed("hnsw: max_level exceeds cap")); + } + if n_nodes > 0 && entry_point as usize >= n_nodes { + return Err(malformed("hnsw: entry_point out of range")); + } Ok(Self { m, m_max0, @@ -132,7 +143,11 @@ fn decode_nodes_v1(cur: &mut ByteCursor, n_nodes: usize) -> Result, Ru } let mut ids = Vec::with_capacity(k); for _ in 0..k { - ids.push(cur.u32()?); + let id = cur.u32()?; + if id as usize >= n_nodes { + return Err(malformed("hnsw v1: neighbour id out of range")); + } + ids.push(id); } neighbors.push(ids); } @@ -161,6 +176,11 @@ fn decode_nodes_v2(cur: &mut ByteCursor, n_nodes: usize) -> Result, Ru if ids.len() != total_ids { return Err(malformed("hnsw v2: neighbour-id column mismatch")); } + // range-check every neighbour id before it is used to index `nodes` / + // the vector store at search time (a hostile id = OOB panic / DoS). + if ids.iter().any(|&id| id as usize >= n_nodes) { + return Err(malformed("hnsw v2: neighbour id out of range")); + } let mut nodes = Vec::with_capacity(n_nodes.min(1 << 16)); let (mut ci, mut ii) = (0usize, 0usize); for &lvl in &levels { @@ -255,4 +275,40 @@ mod tests { // counts and ids columns missing -> EOF, typed error. assert!(HnswIndex::from_bytes(&p, 1, 8).is_err()); } + + // a neighbour id parsed from a hostile file that is >= n_nodes would index + // `nodes` / the vector store out of range at search time. It must be + // rejected at decode, never accepted then panic on the first query. + #[test] + fn v2_rejects_out_of_range_neighbour_id() { + let mut p = v2_header(2); // n_nodes = 2 -> valid ids are {0, 1} + write_blob(&mut p, &pack_u64s(&[0, 0])); // two nodes, level 0 + write_blob(&mut p, &pack_u64s(&[1, 1])); // one neighbour each + write_blob(&mut p, &pack_u64s(&[5, 0])); // id 5 is out of range + assert!(HnswIndex::from_bytes(&p, 2, 8).is_err()); + } + + #[test] + fn v1_rejects_out_of_range_neighbour_id() { + let mut out = 1u32.to_le_bytes().to_vec(); // version 1 + for v in [16u32, 32, 400, 0, 0, 1] { + out.extend_from_slice(&v.to_le_bytes()); // entry 0, max_level 0, n 1 + } + out.extend_from_slice(&0u32.to_le_bytes()); // node 0 level + out.extend_from_slice(&1u32.to_le_bytes()); // layer 0 degree = 1 + out.extend_from_slice(&7u32.to_le_bytes()); // neighbour id 7 >= n(1) + assert!(HnswIndex::from_bytes(&out, 1, 8).is_err()); + } + + #[test] + fn rejects_out_of_range_entry_point() { + let mut out = 2u32.to_le_bytes().to_vec(); // version 2 + for v in [16u32, 32, 400, 9, 0, 1] { + out.extend_from_slice(&v.to_le_bytes()); // entry_point 9, n_nodes 1 + } + write_blob(&mut out, &pack_u64s(&[0])); // one node, level 0 + write_blob(&mut out, &pack_u64s(&[0])); // that layer has 0 neighbours + write_blob(&mut out, &pack_u64s(&[])); // no ids + assert!(HnswIndex::from_bytes(&out, 1, 8).is_err()); + } } diff --git a/crates/nest-runtime/src/bm25/codec.rs b/crates/nest-runtime/src/bm25/codec.rs index 270bf952..804e509e 100644 --- a/crates/nest-runtime/src/bm25/codec.rs +++ b/crates/nest-runtime/src/bm25/codec.rs @@ -94,19 +94,27 @@ fn write_blob(out: &mut Vec, blob: &[u8]) { out.extend_from_slice(blob); } +/// hostile-input guard: `n_docs`, `n_terms`, and `df` are raw `u32` read from +/// an untrusted section and would otherwise be passed straight to +/// `Vec::/HashMap::with_capacity`, letting a crafted header (e.g. `df = +/// 0xFFFF_FFFF`) force a multi-GiB reservation and abort the process at +/// `open()`. Cap the capacity HINT and let the collection grow on demand; the +/// cursor's per-read EOF check bounds the actual work to the bytes present. +const MAX_PREALLOC: usize = 1 << 20; + type Decoded = (Vec, HashMap); /// v1: flat `u32` doc lengths and `(doc, tf)` postings. fn decode_v1(cur: &mut ByteCursor, n_docs: usize, n_terms: usize) -> Result { - let mut doc_lengths = Vec::with_capacity(n_docs); + let mut doc_lengths = Vec::with_capacity(n_docs.min(MAX_PREALLOC)); for _ in 0..n_docs { doc_lengths.push(cur.u32()?); } - let mut terms: HashMap = HashMap::with_capacity(n_terms); + let mut terms: HashMap = HashMap::with_capacity(n_terms.min(MAX_PREALLOC)); for _ in 0..n_terms { let term = cur.term()?; let df = cur.u32()?; - let mut postings = Vec::with_capacity(df as usize); + let mut postings = Vec::with_capacity((df as usize).min(MAX_PREALLOC)); for _ in 0..df { let doc = cur.u32()?; let tf = cur.u32()?; @@ -126,7 +134,7 @@ fn decode_v2(cur: &mut ByteCursor, n_docs: usize, n_terms: usize) -> Result = dls.iter().map(|&x| x as u32).collect(); // term dictionary: (term, df) in alphabetical order. - let mut dict: Vec<(String, u32)> = Vec::with_capacity(n_terms); + let mut dict: Vec<(String, u32)> = Vec::with_capacity(n_terms.min(MAX_PREALLOC)); let mut total: usize = 0; for _ in 0..n_terms { let term = cur.term()?; @@ -139,10 +147,10 @@ fn decode_v2(cur: &mut ByteCursor, n_docs: usize, n_terms: usize) -> Result = HashMap::with_capacity(n_terms); + let mut terms: HashMap = HashMap::with_capacity(n_terms.min(MAX_PREALLOC)); let mut k = 0usize; for (term, df) in dict { - let mut postings = Vec::with_capacity(df as usize); + let mut postings = Vec::with_capacity((df as usize).min(MAX_PREALLOC)); let mut prev = 0u32; for _ in 0..df { let doc = prev.wrapping_add(gaps[k] as u32); @@ -195,3 +203,37 @@ impl<'a> ByteCursor<'a> { unpack_u64s(blob).map_err(RuntimeError::Format) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn v1_header(n_docs: u32, n_terms: u32) -> Vec { + let mut out = 1u32.to_le_bytes().to_vec(); // version 1 + out.extend_from_slice(&1.2f32.to_le_bytes()); // k1 + out.extend_from_slice(&0.75f32.to_le_bytes()); // b + out.extend_from_slice(&1.0f32.to_le_bytes()); // avgdl + out.extend_from_slice(&n_docs.to_le_bytes()); + out.extend_from_slice(&n_terms.to_le_bytes()); + out + } + + // a header claiming ~4.3e9 docs must not reserve ~17 GiB before the read + // loop hits EOF; it must return a typed error, never OOM/abort. + #[test] + fn v1_hostile_n_docs_errors_without_ooming() { + let p = v1_header(u32::MAX, 0); // no doc-length bytes follow + assert!(Bm25Index::from_bytes(&p).is_err()); + } + + // a term declaring df = u32::MAX must not reserve ~34 GiB of postings. + #[test] + fn v1_hostile_df_errors_without_ooming() { + let mut p = v1_header(0, 1); + p.extend_from_slice(&1u32.to_le_bytes()); // term length 1 + p.extend_from_slice(b"a"); // term + p.extend_from_slice(&u32::MAX.to_le_bytes()); // df = u32::MAX + // no posting bytes follow + assert!(Bm25Index::from_bytes(&p).is_err()); + } +} diff --git a/crates/nest-runtime/src/mmap_file.rs b/crates/nest-runtime/src/mmap_file.rs index 28fef415..0d0822a0 100644 --- a/crates/nest-runtime/src/mmap_file.rs +++ b/crates/nest-runtime/src/mmap_file.rs @@ -78,6 +78,10 @@ pub struct MmapNestFile { pub(crate) chunk_ids: Vec, pub(crate) spans: Vec, pub(crate) embedding_model: String, + /// lManifest `model_hash`: the granular fingerprint of the model that + /// produced the corpus embeddings. Exposed so a caller embedding a query + /// can verify its embedder matches the corpus (the honesty gate). + pub(crate) model_hash: String, pub(crate) file_hash: String, pub(crate) content_hash: String, /// lOptional ANN index. Built from the HNSW section payload at open @@ -172,6 +176,7 @@ impl MmapNestFile { }; let embedding_model = view.manifest.embedding_model.clone(); + let model_hash = view.manifest.model_hash.clone(); let declared_index_type = view.manifest.index_type.clone(); let declared_score_type = view.manifest.score_type.clone(); let file_hash = view.file_hash_hex(); @@ -189,6 +194,7 @@ impl MmapNestFile { chunk_ids, spans, embedding_model, + model_hash, file_hash, content_hash, ann_index, @@ -220,6 +226,11 @@ impl MmapNestFile { pub fn declared_index_type(&self) -> &str { &self.declared_index_type } + /// lManifest `model_hash` (the model fingerprint the corpus was built + /// with). Callers embed the query with their own model and compare. + pub fn model_hash(&self) -> &str { + &self.model_hash + } pub fn declared_score_type(&self) -> &str { &self.declared_score_type } diff --git a/crates/nest-runtime/src/simd/avx2.rs b/crates/nest-runtime/src/simd/avx2.rs index ac0a7dc5..8c54e1b2 100644 --- a/crates/nest-runtime/src/simd/avx2.rs +++ b/crates/nest-runtime/src/simd/avx2.rs @@ -145,9 +145,13 @@ pub(super) unsafe fn dot_f32_i8_avx2(q: &[f32], row: &[i8]) -> f32 { let chunks = dim / 8; for i in 0..chunks { let i8_ptr = row.as_ptr().add(i * 8) as *const i64; - // lLoad 8 i8s (8 bytes) into the low half of an xmm. - let raw = _mm_set1_epi64x(*i8_ptr); - // lWiden i8 -> i32 (8 lanes). + // Load 8 i8s (8 bytes) into the low half of an xmm. + // SAFETY: `row` is an `&[i8]` (1-byte alignment), so the `*const + // i64` is not guaranteed 8-byte aligned; a plain `*i8_ptr` deref + // would be UB. `read_unaligned` performs a well-defined unaligned + // load. `chunks = dim/8` keeps `i*8 + 8 <= dim <= row.len()`. + let raw = _mm_set1_epi64x(i8_ptr.read_unaligned()); + // Widen i8 -> i32 (8 lanes). let widened = _mm256_cvtepi8_epi32(raw); let f = _mm256_cvtepi32_ps(widened); let qv = _mm256_loadu_ps(q.as_ptr().add(i * 8)); diff --git a/dat/demo/README.md b/dat/demo/README.md index 8206985a..0f10daaa 100644 --- a/dat/demo/README.md +++ b/dat/demo/README.md @@ -125,3 +125,33 @@ deduplication by sha256 collapses the inevitable overlap (FakeRecogna re-publish ## licenses each subdirectory inherits its upstream license. some are CC-BY-SA, some are MIT, some are unspecified academic distributions. if you redistribute a built `.nest` derived from this directory, the license of the resulting corpus is the union of all upstream licenses (most restrictive wins). check each `README` inside the subdirectories. + +### corpus license bill of materials + +The shipped `dat/corpus_next.v1.nest` embeds text derived from all seven sources +below. **Verify each upstream license before redistributing** — several are +research/academic distributions without an explicit redistribution grant, and at +least one is share-alike (CC-BY-SA), which is viral over the derived corpus. + +| source | upstream | license (verify at source) | redistribution note | +|--------|----------|----------------------------|---------------------| +| `FakeBr-hf` | huggingface `vzani/corpus-fake-br` | see upstream | derived from Fake.br | +| `FakeTrue.Br-hf` | huggingface `vzani/corpus-faketrue-br` | see upstream | | +| `Fake.br-Corpus` | github `roneysco/Fake.br-Corpus` | academic (UFG/USP) — verify grant | attribution expected | +| `FakeTrue.Br` | github `jpchav98/FakeTrue.Br` | academic — verify grant | | +| `FakeRecogna` | github `Gabriel-Lino-Garcia/FakeRecogna` | academic — verify grant | contains health/political claims about named people | +| `factck-br` | github `opit-research/factck-br` (agência lupa) | see upstream — likely attribution/share-alike | fact-check ratings; attribution required | +| `portuguese-fake-news-classifier-bilstm-combined` | huggingface `vzani/...` | see upstream | held-out test split | + +**Obligations for redistributors of a built `.nest`:** + +- Honor the **most-restrictive** upstream license (CC-BY-SA attribution + + share-alike propagates to the whole derived corpus). +- Carry **attribution**: the built file's manifest sets `license` and records + per-source provenance, but `nest cite` does not yet surface a per-chunk + license/attribution field — carry the attribution alongside the artifact until + it does. +- The corpus embeds **personal data about named public figures** (political and + health claims). See [`doc/data-governance.md`](../../doc/data-governance.md) + for the erasure/lawful-basis posture. For anything you ship broadly, prefer the + CC0 `python/forge/demo_corpus` corpus instead of this mixed-license union. diff --git a/doc/data-governance.md b/doc/data-governance.md new file mode 100644 index 00000000..dfb95049 --- /dev/null +++ b/doc/data-governance.md @@ -0,0 +1,76 @@ +# data governance, provenance & licensing + +nest ships a `.nest` as an **immutable, self-contained, content-addressed** +file that is meant to be copied and distributed (phones, edge nodes, air-gapped +boxes). That design has direct governance consequences whenever the embedded +content is personal data. This document records the posture so it is auditable. + +## 1. a shipped `.nest` is a datastore, not a cache + +A `.nest` stores the canonical chunk text and `source_uri` in cleartext, plus +the embeddings (a derived representation of that text). There is no at-rest +encryption in the format — `zstd` is compression, not confidentiality. + +**Consequence:** a `.nest` built over personal or sensitive data (e.g. the +clinical corpus described in [`phi-safety.md`](phi-safety.md)) IS a copy of that +data and must be handled with the same controls as the source, including +**encryption at rest** (FileVault / APFS / LUKS on any volume that holds it). +For sensitive-category data this is a required control, not a "residual risk". + +## 2. right-to-erasure and rectification (LGPD art. 18 / GDPR art. 17, 16) + +The container is immutable and content-addressed: a citation is +`nest:///`, and changing any chunk changes +`content_hash`, invalidating every issued citation and every distributed copy. +The runtime never opens a socket, so there is **no callback channel** to recall +copies already shipped. + +**You cannot honor an erasure/rectification request by editing a distributed +`.nest` in place.** Before shipping any `.nest` that contains personal data: + +1. **Establish a lawful basis and run a DPIA** (GDPR art. 35 / LGPD art. 38). + Prefer to ship only **anonymized or CC0/permissively-licensed** corpora — the + `python/forge/demo_corpus` CC0 path exists for exactly this. +2. **Define a revocation/rotation process** up front, since in-place deletion is + impossible: version the corpus (each build has a distinct `content_hash` / + `file_hash`), publish a revocation list, and place a contractual/operational + obligation on operators to re-pull the current build and destroy superseded + copies. Treat embeddings as in-scope derived personal data. +3. **Record consent/provenance** for embedded third-party content. + +Do not distribute a `.nest` containing special-category data (health, etc.) +without counsel confirming the immutable-distribution model is compatible with +the applicable data-subject rights. + +## 3. provenance & build integrity (as compliance assets) + +These properties make a `.nest` a strong evidentiary artifact: + +- **reproducible builds** — `reproducible=True` + same chunks + same model + fingerprint ⇒ byte-identical `file_hash` on any machine. +- **four SHA-256 hashes** — header, per-section (physical bytes), whole-file, and + `content_hash` (decoded bytes, stable across encodings). +- **model fingerprint** — `model_hash` over config + tokenizer + weights + + pooling + dim + normalize; a query embedded by a different model fails the + honesty gate (CLI, and the Python `retrieve` binding via `expected_model_hash`). + +**Known gaps (tracked hardening items):** + +- Release tags and `nest-cli` binaries are **not yet cryptographically signed**, + and **no SBOM** is published per release. An auditor cannot yet attest + "this artifact == the reviewed source" from a signature. Until signed releases + land, verify artifacts out-of-band. `Cargo.lock` is committed so the Rust + dependency set is pinned; Python deps are declared in `pyproject.toml`. +- The on-disk checksums are unkeyed SHA-256 (integrity, not authenticity); see + [`SECURITY.md`](../SECURITY.md). + +## 4. corpus licensing + +The `.nest` you distribute inherits the license of the content it embeds. The +demo corpus that ships with nest is a **union of several upstream datasets with +different licenses** (some CC-BY-SA, some unspecified academic distributions). +The per-source bill of materials and the effective redistribution obligations +are in [`dat/demo/README.md`](../dat/demo/README.md#corpus-license-bill-of-materials). +The repository code is MIT; that does **not** cover the corpus content. If you +redistribute a built `.nest`, honor the most-restrictive upstream license and +carry attribution. For anything you ship broadly, prefer a permissive/CC0 corpus. diff --git a/python/builder.py b/python/builder.py index 7e182f29..7446f7bf 100644 --- a/python/builder.py +++ b/python/builder.py @@ -95,28 +95,43 @@ def chunk_text( class EmbeddingCache: - """SQLite scratch keyed by chunk_id. Stores raw float32 embeddings. + """SQLite scratch keyed by (chunk_id, model). Stores raw float32 embeddings. Intended to be reused across runs so re-builds skip the expensive embedding step for chunks that have already been computed. + + The key includes `model_key` (embedding_model + model_hash). A chunk_id is + only a function of the text/spans/chunker, NOT the embedding model, so a + cache keyed on chunk_id alone would silently hand back a PREVIOUS model's + vectors when the same corpus is re-embedded with a different model — + shipping a .nest whose vectors do not match its declared model_hash. Keying + on the model closes that. + + Uses table `embeddings_v2`: an old chunk-id-only `embeddings` table (from a + prior nest version) is simply not reused (chunks are re-embedded), never + misread — no in-place migration, no stale-model hit. """ SCHEMA = """ - CREATE TABLE IF NOT EXISTS embeddings ( - chunk_id TEXT PRIMARY KEY, + CREATE TABLE IF NOT EXISTS embeddings_v2 ( + chunk_id TEXT NOT NULL, + model TEXT NOT NULL, dim INTEGER NOT NULL, - data BLOB NOT NULL + data BLOB NOT NULL, + PRIMARY KEY (chunk_id, model) ); """ - def __init__(self, path: str): + def __init__(self, path: str, model_key: str = ""): self.path = path + self.model_key = model_key self.conn = sqlite3.connect(path) self.conn.executescript(self.SCHEMA) def get(self, chunk_id: str, dim: int) -> list[float] | None: row = self.conn.execute( - "SELECT dim, data FROM embeddings WHERE chunk_id=?", (chunk_id,) + "SELECT dim, data FROM embeddings_v2 WHERE chunk_id=? AND model=?", + (chunk_id, self.model_key), ).fetchone() if row is None: return None @@ -127,8 +142,8 @@ def get(self, chunk_id: str, dim: int) -> list[float] | None: def put(self, chunk_id: str, embedding: Sequence[float]) -> None: data = struct.pack(f"<{len(embedding)}f", *embedding) self.conn.execute( - "INSERT OR REPLACE INTO embeddings(chunk_id, dim, data) VALUES(?,?,?)", - (chunk_id, len(embedding), data), + "INSERT OR REPLACE INTO embeddings_v2(chunk_id, model, dim, data) VALUES(?,?,?,?)", + (chunk_id, self.model_key, len(embedding), data), ) self.conn.commit() @@ -195,7 +210,10 @@ def __init__( ): self.cfg = cfg self.embedder = embedder - self.cache = EmbeddingCache(scratch_db) if scratch_db else None + # key the cache by the model too, so re-embedding the same corpus with + # a different model never reuses stale vectors under a new model_hash. + model_key = f"{cfg.embedding_model}\x00{cfg.model_hash}" + self.cache = EmbeddingCache(scratch_db, model_key=model_key) if scratch_db else None self._specs: list[ChunkSpec] = [] def add(self, spec: ChunkSpec) -> None: diff --git a/python/convert_legacy.py b/python/convert_legacy.py index f8ac046a..1dc6afbc 100644 --- a/python/convert_legacy.py +++ b/python/convert_legacy.py @@ -23,6 +23,13 @@ import sys import time +# Force HF/sentence-transformers OFFLINE by default (opt in with +# NEST_ALLOW_DOWNLOAD=1) before any hub access. model_fingerprint also sets +# this on import; kept here too so the guarantee is explicit at the entry point. +if os.environ.get("NEST_ALLOW_DOWNLOAD") != "1": + for _k in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "HF_DATASETS_OFFLINE"): + os.environ.setdefault(_k, "1") + import numpy as np import zstandard as zstd diff --git a/python/embed_query.py b/python/embed_query.py index 492a5c67..a8c756b3 100644 --- a/python/embed_query.py +++ b/python/embed_query.py @@ -34,10 +34,20 @@ import sys from pathlib import Path +# Force HuggingFace/sentence-transformers OFFLINE by default, BEFORE the +# (lazy) sentence_transformers import ever runs. A hostile or misconfigured +# corpus model name must never trigger a hub download mid-run — especially +# while the box is handling PHI (see doc/phi-safety.md). Opt into the +# first-time model fetch explicitly with NEST_ALLOW_DOWNLOAD=1. +if os.environ.get("NEST_ALLOW_DOWNLOAD") != "1": + for _k in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "HF_DATASETS_OFFLINE"): + os.environ.setdefault(_k, "1") + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from model_fingerprint import ( # noqa: E402 compute_model_fingerprint, fingerprint_to_model_hash, + hf_cache_snapshot, ) @@ -81,14 +91,11 @@ def _resolve_local_path(model, fallback: str) -> str: nop = getattr(cfg, "_name_or_path", None) or getattr(cfg, "name_or_path", None) if nop and Path(nop).is_dir(): return str(Path(nop).resolve()) - # hF cache fallback. - hf_home = Path(os.environ.get("HF_HOME", Path.home() / ".cache" / "huggingface")) - cache_dir = hf_home / "hub" / f"models--{fallback.replace('/', '--')}" - snap_dir = cache_dir / "snapshots" - if snap_dir.is_dir(): - snaps = sorted(snap_dir.iterdir()) - if snaps: - return str(snaps[0].resolve()) + # hF cache fallback: resolve the revision refs/main points to (the one + # actually loaded), NOT an arbitrary alphabetical snapshot. + snap = hf_cache_snapshot(fallback) + if snap is not None: + return str(snap) return fallback diff --git a/python/forge/retrieve.py b/python/forge/retrieve.py index bd257196..4292299a 100644 --- a/python/forge/retrieve.py +++ b/python/forge/retrieve.py @@ -26,18 +26,26 @@ import nest # noqa: E402 from builder import BuildConfig, Pipeline, chunk_text # noqa: E402 + from forge.embed_potion import potion_embedder # noqa: E402 CORPUS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "demo_corpus") -def retrieve(nestfile, query: str, k: int = 5, embedder=None): +def retrieve(nestfile, query: str, k: int = 5, embedder=None, verify_model: bool = True): """embed `query` OFFLINE with potion, then NestFile.retrieve. the returned hits' `score` is the exact-cosine rerank value; each carries tier-1 text + - a nest:// citation. `embedder` defaults to the vendored potion table.""" + a nest:// citation. `embedder` defaults to the vendored potion table. + + honesty gate (on by default): the embedder's `model_hash` is passed to + NestFile.retrieve, which raises if it does not match the corpus manifest — + so a query embedded by a DIFFERENT model can never silently return + cosine-valid, wrong hits (the invariant the CLI enforces, now on the + flagship Python surface too). Set `verify_model=False` to bypass.""" emb = embedder or potion_embedder() qvec = emb.embed_texts([query])[0] - return nestfile.retrieve(qvec, k) + expected = emb.model_hash() if (verify_model and hasattr(emb, "model_hash")) else None + return nestfile.retrieve(qvec, k, expected_model_hash=expected) def build_demo(out_path: str) -> str: diff --git a/python/forge/test_retrieve.py b/python/forge/test_retrieve.py index 2f3675bd..28fd241a 100644 --- a/python/forge/test_retrieve.py +++ b/python/forge/test_retrieve.py @@ -24,6 +24,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # python/ import nest # noqa: E402 + from forge.embed_potion import potion_embedder # noqa: E402 from forge.retrieve import build_demo, retrieve # noqa: E402 diff --git a/python/model_fingerprint.py b/python/model_fingerprint.py index c0470f09..52241c45 100644 --- a/python/model_fingerprint.py +++ b/python/model_fingerprint.py @@ -29,9 +29,16 @@ import hashlib import json +import os from dataclasses import dataclass from pathlib import Path +# Force HF/sentence-transformers OFFLINE by default before any (lazy) hub +# access. Opt into a first-time model download with NEST_ALLOW_DOWNLOAD=1. +if os.environ.get("NEST_ALLOW_DOWNLOAD") != "1": + for _k in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "HF_DATASETS_OFFLINE"): + os.environ.setdefault(_k, "1") + # lFiles whose contents affect inference output. Other files in the # lsnapshot dir (LICENSE, README.md, .git*) are ignored. RELEVANT_FILES: tuple[str, ...] = ( @@ -176,6 +183,36 @@ def is_placeholder(model_hash: str) -> bool: return model_hash == PLACEHOLDER_HASH +def hf_cache_snapshot(model_id: str) -> Path | None: + """Resolve the HF Hub cache snapshot dir for `model_id`, picking the + revision that `refs/main` points to — the one HF actually loads — rather + than an arbitrary alphabetical snapshot. + + Returns `None` if the model is not cached, or if the loaded revision + cannot be determined unambiguously (multiple snapshots and no usable + `refs/main`). Callers must then fail closed / require an explicit + `--model-path`, never guess: hashing the wrong snapshot would stamp a + corpus with a `model_hash` that does not match the model that produced + its vectors, defeating the honesty gate. + """ + hf_home = Path(os.environ.get("HF_HOME", Path.home() / ".cache" / "huggingface")) + cache_dir = hf_home / "hub" / f"models--{model_id.replace('/', '--')}" + snap_dir = cache_dir / "snapshots" + if not snap_dir.is_dir(): + return None + ref = cache_dir / "refs" / "main" + if ref.is_file(): + commit = ref.read_text().strip() + cand = snap_dir / commit + if cand.is_dir(): + return cand.resolve() + snaps = [d for d in snap_dir.iterdir() if d.is_dir()] + if len(snaps) == 1: + return snaps[0].resolve() + # zero snapshots, or ambiguous (multiple with no usable ref): do not guess. + return None + + def resolve_model_dir(model_name_or_path: str) -> Path: """Resolve a model name (HF id) or local path to its snapshot dir. @@ -192,21 +229,16 @@ def resolve_model_dir(model_name_or_path: str) -> Path: Raises `FileNotFoundError` with an actionable message if nothing works — the caller should pass `--model-path` explicitly. """ - import os as _os - p = Path(model_name_or_path).expanduser() if p.is_dir(): return p.resolve() - # hF Hub cache: standard layout. Finds the model regardless of - # whether sentence-transformers is importable. - hf_home = Path(_os.environ.get("HF_HOME", Path.home() / ".cache" / "huggingface")) - cache_dir = hf_home / "hub" / f"models--{model_name_or_path.replace('/', '--')}" - snap_dir = cache_dir / "snapshots" - if snap_dir.is_dir(): - snaps = sorted(snap_dir.iterdir()) - if snaps: - return snaps[0].resolve() + # hF Hub cache: standard layout, resolving the revision refs/main points + # to (not an arbitrary alphabetical snapshot). Finds the model regardless + # of whether sentence-transformers is importable. + snap = hf_cache_snapshot(model_name_or_path) + if snap is not None: + return snap # older sentence-transformers (v2.x) put the snapshot path in the # autoModel config. Importing is heavy; only do it as a last resort. diff --git a/python/nest.py b/python/nest.py index 5697e698..36a93fd5 100644 --- a/python/nest.py +++ b/python/nest.py @@ -41,7 +41,8 @@ def _find_extension() -> str | None: if _ext_path is None: raise ImportError( "Cannot find _nest extension. Run " - "`cargo build --release -p nest-python && cp target/release/lib_nest.dylib python/_nest.so` " + "`cargo build --release -p nest-python && " + "cp target/release/lib_nest.dylib python/_nest.so` " "from the repo root." ) diff --git a/python/tools/nest_build_corpus.py b/python/tools/nest_build_corpus.py index fc0ac421..9877dbd5 100644 --- a/python/tools/nest_build_corpus.py +++ b/python/tools/nest_build_corpus.py @@ -21,11 +21,20 @@ import argparse import hashlib +import os import subprocess import sys from collections import Counter from pathlib import Path +# Force HF/sentence-transformers OFFLINE by default before the (lazy) ST +# import at embed time. A corpus build touches source text; it must not make +# an unexpected outbound fetch. Opt into a first-time download with +# NEST_ALLOW_DOWNLOAD=1 (then re-run offline for reproducibility). +if os.environ.get("NEST_ALLOW_DOWNLOAD") != "1": + for _k in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "HF_DATASETS_OFFLINE"): + os.environ.setdefault(_k, "1") + REPO = Path(__file__).resolve().parents[2] sys.path.insert(0, str(Path(__file__).resolve().parent)) sys.path.insert(0, str(REPO / "python")) diff --git a/tests/test_builder.py b/tests/test_builder.py index 3d14364c..45f5ec6e 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -1,4 +1,5 @@ """end-to-end test of the Python ingestion pipeline.""" + import math import os import sys @@ -30,7 +31,7 @@ def test_chunk_text_byte_spans_round_trip(): assert rebuilt == encoded, (rebuilt, encoded) # Byte spans must point to the right place in the original encoding. for c in chunks: - assert encoded[c.byte_start:c.byte_end] == c.canonical_text.encode("utf-8") + assert encoded[c.byte_start : c.byte_end] == c.canonical_text.encode("utf-8") def test_pipeline_emits_validated_nest_file(): @@ -135,7 +136,7 @@ def test_mrl_truncation_sets_embedding_dim_and_query_stride(): byte_end=s.byte_end, embedding=e, ) - for s, e in zip(specs, embs) + for s, e in zip(specs, embs, strict=False) ] nest.build( output_path=out, @@ -198,10 +199,82 @@ def test_mrl_dim_validation_rejects_oversized_and_zero(): assert raised, f"mrl_dim={bad} must be rejected" +def test_cache_keyed_by_model_no_stale_reuse(): + """EmbeddingCache must not hand back a DIFFERENT model's vectors for the + same chunk_id (audit finding P2). chunk_id is model-independent, so a + cache keyed on chunk_id alone would silently reuse stale vectors.""" + with tempfile.TemporaryDirectory() as d: + scratch = os.path.join(d, "cache.db") + a = EmbeddingCache(scratch, model_key="modelA") + a.put("chunk1", [0.1, 0.2, 0.3, 0.4]) + got = a.get("chunk1", 4) + assert got is not None and abs(got[0] - 0.1) < 1e-6, got + a.close() + + # a different model must MISS, even though the chunk_id is identical. + b = EmbeddingCache(scratch, model_key="modelB") + assert b.get("chunk1", 4) is None, "different model must not reuse vectors" + b.close() + + # the original model still hits. + a2 = EmbeddingCache(scratch, model_key="modelA") + again = a2.get("chunk1", 4) + assert again is not None and len(again) == 4 + a2.close() + + +def test_retrieve_model_hash_gate(): + """The Python flagship retrieve binding enforces the model_hash honesty + gate when the caller passes expected_model_hash (audit finding S4).""" + h1 = "sha256:" + "ab" * 32 + wrong = "sha256:" + "cd" * 32 + with tempfile.TemporaryDirectory() as d: + out = os.path.join(d, "gate.nest") + chunks = [ + dict( + canonical_text="alpha", + source_uri="a.txt", + byte_start=0, + byte_end=5, + embedding=[1.0, 0.0, 0.0, 0.0], + ), + dict( + canonical_text="beta", + source_uri="b.txt", + byte_start=5, + byte_end=9, + embedding=[0.0, 1.0, 0.0, 0.0], + ), + ] + nest.build( + output_path=out, + embedding_model="toy", + embedding_dim=4, + chunker_version="char/512", + model_hash=h1, + chunks=chunks, + reproducible=True, + ) + db = nest.open(out) + assert db.model_hash == h1, db.model_hash + q = [1.0, 0.0, 0.0, 0.0] + assert db.retrieve(q, 1, expected_model_hash=h1), "matching hash must succeed" + assert db.retrieve(q, 1), "no expected hash must stay backward-compatible" + raised = False + try: + db.retrieve(q, 1, expected_model_hash=wrong) + except ValueError as e: + raised = True + assert "model_hash mismatch" in str(e), str(e) + assert raised, "a mismatched expected_model_hash must raise" + + if __name__ == "__main__": test_chunk_text_byte_spans_round_trip() test_pipeline_emits_validated_nest_file() test_cache_skips_re_embedding_on_second_run() + test_cache_keyed_by_model_no_stale_reuse() + test_retrieve_model_hash_gate() test_mrl_truncation_sets_embedding_dim_and_query_stride() test_mrl_dim_validation_rejects_oversized_and_zero() print("builder tests OK") diff --git a/tests/test_offline_guard.py b/tests/test_offline_guard.py new file mode 100644 index 00000000..095f3836 --- /dev/null +++ b/tests/test_offline_guard.py @@ -0,0 +1,53 @@ +"""Guard test: the sentence-transformers entrypoints force HuggingFace +OFFLINE by default, and honor NEST_ALLOW_DOWNLOAD=1 as the explicit opt-in +(audit findings S5 / P1). + +Importing `embed_query` must set HF_HUB_OFFLINE=1 before any hub access, so a +hostile/misconfigured corpus model name can never trigger a download mid-run +(e.g. while the box is handling PHI). Runs in a subprocess with a clean env so +the module-load-time guard is observed in isolation. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +PYDIR = str(Path(__file__).resolve().parent.parent / "python") +SNIPPET = ( + "import os, sys; sys.path.insert(0, os.environ['PYDIR']); " + "import embed_query; print(os.environ.get('HF_HUB_OFFLINE'))" +) + + +def _run(extra_env: dict) -> str: + env = dict(os.environ) + env["PYDIR"] = PYDIR + # clean slate: the guard uses setdefault, so a pre-set value would mask it. + _forced = ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE", "HF_DATASETS_OFFLINE") + for k in (*_forced, "NEST_ALLOW_DOWNLOAD"): + env.pop(k, None) + env.update(extra_env) + proc = subprocess.run([sys.executable, "-c", SNIPPET], capture_output=True, text=True, env=env) + assert proc.returncode == 0, proc.stderr + return proc.stdout.strip() + + +def test_offline_forced_by_default() -> None: + assert _run({}) == "1", "HF_HUB_OFFLINE must be forced to 1 by default" + print("offline forced by default: ok") + + +def test_opt_in_download_disables_force() -> None: + assert _run({"NEST_ALLOW_DOWNLOAD": "1"}) == "None", ( + "NEST_ALLOW_DOWNLOAD=1 must NOT force offline (explicit opt-in)" + ) + print("NEST_ALLOW_DOWNLOAD opt-in respected: ok") + + +if __name__ == "__main__": + test_offline_forced_by_default() + test_opt_in_download_disables_force() + print("offline guard tests OK") From 04d07e556d6d4bff239437cb755deaa10e79f780 Mon Sep 17 00:00:00 2001 From: Brenner Cruvinel Date: Wed, 1 Jul 2026 14:24:20 -0300 Subject: [PATCH 2/3] cargo fmt: wrap two over-width lines to satisfy fmt --check gate --- crates/nest-format/src/encoding/zstd_codec.rs | 5 ++++- crates/nest-python/src/nest_file.rs | 10 +++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/crates/nest-format/src/encoding/zstd_codec.rs b/crates/nest-format/src/encoding/zstd_codec.rs index 97f523c9..36d405e5 100644 --- a/crates/nest-format/src/encoding/zstd_codec.rs +++ b/crates/nest-format/src/encoding/zstd_codec.rs @@ -84,6 +84,9 @@ mod tests { fn cap_has_floor_and_ratio() { assert_eq!(decompress_cap(0), MIN_DECOMPRESS_CAP); assert_eq!(decompress_cap(usize::MAX), usize::MAX); // saturating, no overflow - assert_eq!(decompress_cap(MIN_DECOMPRESS_CAP), MIN_DECOMPRESS_CAP * MAX_DECOMPRESS_RATIO); + assert_eq!( + decompress_cap(MIN_DECOMPRESS_CAP), + MIN_DECOMPRESS_CAP * MAX_DECOMPRESS_RATIO + ); } } diff --git a/crates/nest-python/src/nest_file.rs b/crates/nest-python/src/nest_file.rs index 7f14734c..00a19442 100644 --- a/crates/nest-python/src/nest_file.rs +++ b/crates/nest-python/src/nest_file.rs @@ -99,7 +99,15 @@ impl NestFile { ef: usize, expected_model_hash: Option, ) -> PyResult> { - crate::retrieve_fn::retrieve(&self.rt, query, k, candidates, hops, ef, expected_model_hash) + crate::retrieve_fn::retrieve( + &self.rt, + query, + k, + candidates, + hops, + ef, + expected_model_hash, + ) } #[getter] From 9ecf82cb2f840a1debd7b61ac3f5cfcebd5b5eee Mon Sep 17 00:00:00 2001 From: Brenner Cruvinel Date: Wed, 1 Jul 2026 14:32:28 -0300 Subject: [PATCH 3/3] avx2: allow(unused_unsafe) on the register-only srai helper (clippy -D warnings gate) --- crates/nest-runtime/src/simd/avx2.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/nest-runtime/src/simd/avx2.rs b/crates/nest-runtime/src/simd/avx2.rs index 8c54e1b2..c73dc644 100644 --- a/crates/nest-runtime/src/simd/avx2.rs +++ b/crates/nest-runtime/src/simd/avx2.rs @@ -82,6 +82,12 @@ pub(super) unsafe fn dot_f32_i4_avx2( /// lArithmetic shift right by 4 on packed i8 lanes (AVX2 has no epi8 SRA), /// emulated via the epi16 SRA plus a byte-wise blend of even/odd lanes. #[target_feature(enable = "avx2,fma")] +// This body is pure register-only intrinsics (no memory I/O). On our MSRV +// (1.85) those are `unsafe` and the block is required under edition 2024's +// `unsafe_op_in_unsafe_fn`; newer toolchains reclassify them as safe under the +// enabled target feature, making the block redundant (`unused_unsafe`). Allow +// the lint so the same source compiles clean on both. +#[allow(unused_unsafe)] unsafe fn mm_srai_epi8_4(v: std::arch::x86_64::__m128i) -> std::arch::x86_64::__m128i { unsafe { use std::arch::x86_64::*;