diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 9b07a7e..3466517 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -14,10 +14,12 @@ jobs: test: runs-on: ${{matrix.platform}} strategy: + fail-fast: false matrix: include: - {platform: ubuntu-latest, rust: nightly} - {platform: ubuntu-latest, rust: stable} + - {platform: ubuntu-24.04-arm, rust: stable} - {platform: windows-latest, rust: nightly} - {platform: macos-latest, rust: nightly} steps: @@ -43,9 +45,9 @@ jobs: pip install pytest pytest-benchmark dask pytest -v --log-cli-level=debug -s - name: Upload wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: wheels + name: wheels-${{matrix.platform}}-${{matrix.rust}} path: dist release: @@ -54,9 +56,10 @@ jobs: if: "startsWith(github.ref, 'refs/tags/')" needs: [ test ] steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: - name: wheels + pattern: wheels-* + merge-multiple: true - uses: actions/setup-python@v4 with: python-version: "3.10" diff --git a/Cargo.lock b/Cargo.lock index 0b4e00a..ea35b84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,7 +112,18 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.87", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", ] [[package]] @@ -121,12 +132,77 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "attohttpc" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e2cdb6d5ed835199484bb92bb8b3edd526effe995c61732580439c1a67e2e9" +dependencies = [ + "base64", + "http", + "log", + "rustls", + "serde", + "serde_json", + "url", + "webpki-roots 1.0.8", +] + [[package]] name = "autocfg" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-creds" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3b85155d265df828f84e53886ed9e427aed979dd8a39f5b8b2162c77e142d7" +dependencies = [ + "attohttpc", + "home", + "log", + "quick-xml", + "rust-ini", + "serde", + "thiserror", + "time", + "url", +] + +[[package]] +name = "aws-lc-rs" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "aws-region" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "838b36c8dc927b6db1b6c6b8f5d05865f2213550b9e83bf92fa99ed6525472c0" +dependencies = [ + "thiserror", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -169,6 +245,15 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -195,10 +280,13 @@ checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "cc" -version = "1.1.28" +version = "1.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e80e3b6a3ab07840e1cae9b0666a63970dc28e8ed5ffbcdacbfc760c281bfc1" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -208,6 +296,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "clap" version = "4.5.20" @@ -240,7 +334,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.87", ] [[package]] @@ -251,9 +345,9 @@ checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] name = "cmake" -version = "0.1.51" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" dependencies = [ "cc", ] @@ -270,6 +364,26 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af" +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.15", + "once_cell", + "tiny-keccak", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -286,6 +400,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -320,6 +443,22 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "curl-sys" version = "0.4.77+curl-8.10.1" @@ -335,6 +474,26 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "serde_core", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + [[package]] name = "divan" version = "0.1.14" @@ -357,9 +516,24 @@ checksum = "27540baf49be0d484d8f0130d7d8da3011c32a44d4fc873368154f1510e574a2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.87", +] + +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "either" version = "1.13.0" @@ -397,6 +571,12 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + [[package]] name = "flexbuffers" version = "2.0.0" @@ -456,6 +636,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.31" @@ -512,7 +698,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.87", ] [[package]] @@ -554,6 +740,16 @@ dependencies = [ "byteorder", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -561,8 +757,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", ] [[package]] @@ -590,6 +802,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "hashbrown" version = "0.15.0" @@ -628,7 +846,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.87", ] [[package]] @@ -689,6 +907,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hidefix" version = "0.12.0" @@ -716,9 +940,10 @@ dependencies = [ "netcdf", "numpy", "pyo3", - "rand", + "rand 0.8.5", "rayon", "reqwest", + "rust-s3", "serde", "serde_bytes", "sled", @@ -727,6 +952,24 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "http" version = "1.1.0" @@ -802,6 +1045,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", + "webpki-roots 0.26.11", ] [[package]] @@ -856,7 +1100,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.0", ] [[package]] @@ -901,12 +1145,24 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" -version = "0.3.71" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cb94a0ffd3f3ee755c20f7d8752f45cac88605a4dcf808abcff72873296ec7b" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ + "cfg-if", + "futures-util", "wasm-bindgen", ] @@ -918,9 +1174,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.159" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libdeflate-sys" @@ -947,7 +1203,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] @@ -990,9 +1246,15 @@ version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown", + "hashbrown 0.15.0", ] +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "matrixmultiply" version = "0.3.9" @@ -1003,6 +1265,23 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "maybe-async" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "746873a384ad60adc5db74471dfaba74bd278afbdcfd81db93fafcdfc8b5ca0c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "md5" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" + [[package]] name = "memchr" version = "2.7.4" @@ -1130,6 +1409,15 @@ dependencies = [ "semver", ] +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -1139,6 +1427,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-integer" version = "0.1.46" @@ -1190,7 +1484,26 @@ dependencies = [ "num-integer", "num-traits", "pyo3", - "rustc-hash", + "rustc-hash 1.1.0", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.6.0", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", ] [[package]] @@ -1231,7 +1544,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.87", ] [[package]] @@ -1262,6 +1575,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.5", +] + [[package]] name = "parking_lot" version = "0.11.2" @@ -1355,6 +1678,12 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -1463,7 +1792,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.79", + "syn 2.0.87", ] [[package]] @@ -1476,7 +1805,72 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.79", + "syn 2.0.87", +] + +[[package]] +name = "quick-xml" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash 2.1.2", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash 2.1.2", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.59.0", ] [[package]] @@ -1488,6 +1882,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "rand" version = "0.8.5" @@ -1495,8 +1895,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", ] [[package]] @@ -1506,7 +1916,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", ] [[package]] @@ -1515,7 +1935,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", ] [[package]] @@ -1625,7 +2054,10 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -1633,11 +2065,15 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", + "tokio-rustls", + "tokio-util", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", + "webpki-roots 0.26.11", "windows-registry", ] @@ -1649,13 +2085,57 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.15", "libc", "spin", "untrusted", "windows-sys 0.52.0", ] +[[package]] +name = "rust-ini" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + +[[package]] +name = "rust-s3" +version = "0.37.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeedb13abdaa7e48d391de05b0569b37fa0a7a64a668dff6ffb2141ad0c2527e" +dependencies = [ + "async-trait", + "aws-creds", + "aws-region", + "base64", + "bytes", + "cfg-if", + "futures-util", + "hex", + "hmac", + "http", + "log", + "maybe-async", + "md5", + "percent-encoding", + "quick-xml", + "reqwest", + "serde", + "serde_derive", + "serde_json", + "sha2", + "sysinfo", + "thiserror", + "time", + "tokio", + "tokio-stream", + "url", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -1668,6 +2148,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + [[package]] name = "rustix" version = "0.38.37" @@ -1683,11 +2169,14 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.14" +version = "0.23.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" +checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" dependencies = [ + "aws-lc-rs", + "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -1705,21 +2194,32 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.9.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +dependencies = [ + "web-time", + "zeroize", +] [[package]] name = "rustls-webpki" -version = "0.102.8" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", ] +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + [[package]] name = "ryu" version = "1.0.18" @@ -1772,10 +2272,11 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] @@ -1788,15 +2289,24 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.87", ] [[package]] @@ -1823,11 +2333,22 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "slab" @@ -1907,9 +2428,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.79" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -1925,6 +2446,20 @@ dependencies = [ "futures-core", ] +[[package]] +name = "sysinfo" +version = "0.37.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16607d5caffd1c07ce073528f9ed972d88db15dd44023fa57142963be3feb11f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + [[package]] name = "system-configuration" version = "0.6.1" @@ -1975,6 +2510,65 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "time" +version = "0.3.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -2014,7 +2608,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.87", ] [[package]] @@ -2038,6 +2632,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.12" @@ -2110,6 +2715,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-bidi" version = "0.3.17" @@ -2188,48 +2799,42 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] -name = "wasm-bindgen" -version = "0.2.94" +name = "wasip2" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef073ced962d62984fb38a36e5fdc1a2b23c9e0e1fa0689bb97afa4202ef6887" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ - "cfg-if", - "once_cell", - "wasm-bindgen-macro", + "wit-bindgen", ] [[package]] -name = "wasm-bindgen-backend" -version = "0.2.94" +name = "wasm-bindgen" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4bfab14ef75323f4eb75fa52ee0a3fb59611977fd3240da19b2cf36ff85030e" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ - "bumpalo", - "log", + "cfg-if", "once_cell", - "proc-macro2", - "quote", - "syn 2.0.79", + "rustversion", + "wasm-bindgen-macro", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.44" +version = "0.4.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65471f79c1022ffa5291d33520cbbb53b7687b01c2f8e83b57d102eed7ed479d" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" dependencies = [ - "cfg-if", "js-sys", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.94" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7bec9830f60924d9ceb3ef99d55c155be8afa76954edffbb5936ff4509474e7" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2237,33 +2842,77 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.94" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c74f6e152a76a2ad448e223b0fc0b6b5747649c3d769cc6bf45737bf97d0ed6" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ + "bumpalo", "proc-macro2", "quote", - "syn 2.0.79", - "wasm-bindgen-backend", + "syn 2.0.87", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.94" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a42f6c679374623f295a8623adfe63d9284091245c3504bde47c17a3ce2777d9" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] [[package]] name = "web-sys" -version = "0.3.71" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44188d185b5bdcae1052d08bcbcf9091a5524038d4572cc4f4f2bb9d5554ddd9" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ "js-sys", "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.8", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -2286,14 +2935,104 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-link 0.1.3", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.4.2", +] + +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core", + "windows-link 0.1.3", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core", + "windows-link 0.1.3", +] + [[package]] name = "windows-registry" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" dependencies = [ - "windows-result", - "windows-strings", + "windows-result 0.2.0", + "windows-strings 0.1.0", "windows-targets 0.52.6", ] @@ -2306,16 +3045,34 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-strings" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" dependencies = [ - "windows-result", + "windows-result 0.2.0", "windows-targets 0.52.6", ] +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -2343,6 +3100,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link 0.2.1", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -2374,6 +3140,15 @@ dependencies = [ "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -2493,6 +3268,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "zerocopy" version = "0.7.35" @@ -2511,7 +3292,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.87", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 156a0e1..5dd39f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,9 +37,10 @@ rayon = "1.10" ndarray = { version = "0.16", features = [ "rayon" ] } # Remove when https://github.com/PyO3/rust-numpy/pull/439 is addressed ndarray_0_15 = { package = "ndarray", version = "0.15", features = ["rayon"] } -pyo3 = { version = "0.21", optional = true, features = ["anyhow", "auto-initialize", "abi3-py39"] } +pyo3 = { version = "0.21", optional = true, features = ["anyhow", "abi3-py39"] } numpy = { version = "0.21.0", optional = true } netcdf = { version = "0.10.4", optional = true } +rust-s3 = { version = "0.37", default-features = false, optional = true, features = ["tokio-rustls-tls"] } serde = { version = "1", features = ["derive"] } tokio = { version = "1", features = ["sync", "macros", "rt-multi-thread"] } clap = { version = "4.5.16", features = ["derive"], optional = true } @@ -67,6 +68,7 @@ extension-module = ["python", "pyo3/extension-module"] unstable = [] flexbuffers = ["dep:flexbuffers"] clap = ["dep:clap"] +s3 = ["dep:rust-s3"] bincode = ["dep:bincode"] [[bin]] diff --git a/src/python.rs b/src/python.rs index 430cb1b..6d43820 100644 --- a/src/python.rs +++ b/src/python.rs @@ -262,9 +262,17 @@ mod tests { use super::*; use pyo3::types::PyFloat; + /// The interpreter is initialized explicitly (rather than through pyo3's + /// `auto-initialize` feature, which breaks wheel builds against statically linked + /// pythons like the manylinux ones). + fn with_gil R, R>(f: F) -> R { + pyo3::prepare_freethreaded_python(); + Python::with_gil(f) + } + #[test] fn coads_slice() { - Python::with_gil(|py| { + with_gil(|py| { let i = Index::new("tests/data/coads_climatology.nc4".into()).unwrap(); let ds = i.dataset("SST", None).unwrap(); @@ -275,7 +283,7 @@ mod tests { #[test] fn coads_index_slice() { - Python::with_gil(|py| { + with_gil(|py| { let i = Index::new("tests/data/coads_climatology.nc4".into()).unwrap(); let ds = i.dataset("SST", None).unwrap(); @@ -286,7 +294,7 @@ mod tests { #[test] fn fill_value() { - Python::with_gil(|py| { + with_gil(|py| { let i = Index::new("tests/data/coads_climatology.nc4".into()).unwrap(); let ds = i.dataset("SST", None).unwrap(); diff --git a/src/reader.rs b/src/reader.rs index 8413c3e..354c97d 100644 --- a/src/reader.rs +++ b/src/reader.rs @@ -2,6 +2,8 @@ pub mod cache; pub(crate) mod chunk; pub mod dataset; pub mod direct; +#[cfg(feature = "s3")] +pub mod s3; pub mod stream; pub use dataset::{ParReader, ParReaderExt, Reader, ReaderExt, Streamer, StreamerExt}; diff --git a/src/reader/s3.rs b/src/reader/s3.rs new file mode 100644 index 0000000..84369c5 --- /dev/null +++ b/src/reader/s3.rs @@ -0,0 +1,309 @@ +//! Reader fetching chunks from S3 (or any S3-compatible object store). +//! +//! Chunks are fetched with ranged `GET`s: requests are made asynchronously (several in +//! flight at a time) while decompression is done synchronously as the chunks arrive. +//! Chunks that are (close to) adjacent in the file are coalesced into a single request. +//! +//! The [`Bucket`] is configured by the caller (region, endpoint, credentials), the reader +//! only issues ranged `GET`s for the object key against it: +//! +//! ```no_run +//! use hidefix::prelude::*; +//! use hidefix::idx::DatasetD; +//! use hidefix::reader::s3::S3Reader; +//! +//! let i = Index::index("tests/data/coads_climatology.nc4").unwrap(); +//! let DatasetD::D3(ds) = i.dataset("SST").unwrap() else { panic!() }; +//! +//! let bucket = s3::Bucket::new( +//! "my-bucket", +//! "eu-west-1".parse().unwrap(), +//! s3::creds::Credentials::default().unwrap(), +//! ).unwrap(); +//! let mut r = S3Reader::with_dataset(ds, bucket, "coads_climatology.nc4").unwrap(); +//! +//! let values = r.values::(..).unwrap(); +//! ``` +use std::future::Future; + +use anyhow::ensure; +use futures::{StreamExt, TryStreamExt}; +use s3::Bucket; + +use super::{chunk::decode_chunk, dataset::Reader}; +use crate::extent::Extents; +use crate::filters::byteorder::Order; +use crate::idx::{Chunk, Dataset}; + +/// Maximum number of concurrent range requests. +const CONCURRENT_REQUESTS: usize = 16; + +/// Chunks closer together than this (bytes) are coalesced into a single range request, +/// the gap is fetched and discarded. +const COALESCE_GAP: u64 = 32 * 1024; + +/// Maximum size (bytes) of a coalesced range request. +const MAX_REQUEST_SZ: u64 = 8 * 1024 * 1024; + +pub struct S3Reader<'a, const D: usize> { + ds: &'a Dataset<'a, D>, + bucket: Box, + key: String, + chunk_sz: u64, +} + +/// Segments of a chunk: destination offset, start and end (in the same units as +/// [`Dataset::group_chunk_slices`]). +type Segments = Vec<(u64, u64, u64)>; + +/// A single ranged `GET` covering one or more chunks. +struct Request<'a, const D: usize> { + /// Byte address of first chunk. + start: u64, + + /// One past the last byte of the last chunk. + end: u64, + + /// Chunks in request with their segments. + chunks: Vec<(&'a Chunk, Segments)>, +} + +/// Group chunk slices (sorted by file address, as returned by +/// [`Dataset::group_chunk_slices`]) into coalesced range requests. +fn group_requests<'a, const D: usize>( + groups: Vec<(&'a Chunk, u64, u64, u64)>, +) -> Vec> { + let mut requests: Vec> = Vec::new(); + + for (c, current, start, end) in groups { + let addr = c.addr.get(); + let sz = c.size.get(); + + match requests.last_mut() { + Some(r) if r.chunks.last().unwrap().0.addr == c.addr => { + // Another slice of the previous chunk. + r.chunks.last_mut().unwrap().1.push((current, start, end)); + } + Some(r) + if addr >= r.end + && addr - r.end <= COALESCE_GAP + && (addr + sz) - r.start <= MAX_REQUEST_SZ => + { + // Close enough to the previous chunk: coalesce into the same request. + r.end = addr + sz; + r.chunks.push((c, vec![(current, start, end)])); + } + _ => requests.push(Request { + start: addr, + end: addr + sz, + chunks: vec![(c, vec![(current, start, end)])], + }), + } + } + + requests +} + +/// Drive a future to completion from a blocking context, whether or not we are already +/// inside a tokio runtime. +fn block_on(fut: F) -> F::Output +where + F::Output: Send, +{ + static RUNTIME: std::sync::OnceLock = std::sync::OnceLock::new(); + let runtime = || RUNTIME.get_or_init(|| tokio::runtime::Runtime::new().unwrap()); + + match tokio::runtime::Handle::try_current() { + Ok(handle) if handle.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread => { + tokio::task::block_in_place(|| handle.block_on(fut)) + } + // `block_in_place` panics on a current-thread runtime: drive the future on the + // fallback runtime from a scoped thread instead. + Ok(_) => std::thread::scope(|scope| { + scope + .spawn(|| runtime().block_on(fut)) + .join() + .expect("block_on thread panicked") + }), + Err(_) => runtime().block_on(fut), + } +} + +impl<'a, const D: usize> S3Reader<'a, D> { + pub fn with_dataset>( + ds: &'a Dataset, + bucket: Box, + key: S, + ) -> Result, anyhow::Error> { + let chunk_sz = ds.chunk_shape.iter().product::() * ds.dsize as u64; + + Ok(S3Reader { + ds, + bucket, + key: key.into(), + chunk_sz, + }) + } + + /// Reads raw bytes of slice into destination buffer, async version of + /// [`Reader::read_to`]. Range requests are made concurrently, decompression is done + /// synchronously as the chunks arrive. + pub async fn read_to_async( + &self, + extents: &Extents, + dst: &mut [u8], + ) -> Result { + let dsz = self.ds.dsize as u64; + let counts = extents.get_counts(&self.ds.shape)?; + let vsz = counts.product::() * dsz; + + ensure!( + dst.len() >= vsz as usize, + "destination buffer has insufficient capacity" + ); + + async fn fetch<'r, const D: usize>( + bucket: &Bucket, + key: &str, + r: Request<'r, D>, + ) -> Result<(Request<'r, D>, bytes::Bytes), anyhow::Error> { + let resp = bucket + .get_object_range(key, r.start, Some(r.end - 1)) + .await?; + // A `200` means the server ignored the `Range` header and returned the full + // object: the body only lines up with our offsets when the request starts at + // the beginning of the object. + ensure!( + resp.status_code() == 206 || (resp.status_code() == 200 && r.start == 0), + "range request failed: status code: {}", + resp.status_code() + ); + let bytes = resp.into_bytes(); + ensure!( + bytes.len() as u64 >= r.end - r.start, + "range request returned too few bytes: {} < {}", + bytes.len(), + r.end - r.start + ); + Ok((r, bytes)) + } + + let requests = group_requests(self.ds.group_chunk_slices(extents)) + .into_iter() + .map(|r| fetch(&self.bucket, &self.key, r)) + .collect::>(); + + let mut responses = futures::stream::iter(requests).buffer_unordered(CONCURRENT_REQUESTS); + + while let Some((request, bytes)) = responses.try_next().await? { + for (c, segments) in request.chunks { + let offset = (c.addr.get() - request.start) as usize; + let chunk = bytes[offset..(offset + c.size.get() as usize)].to_vec(); + + let cache = decode_chunk( + chunk, + self.chunk_sz, + dsz, + self.ds.gzip.is_some(), + self.ds.shuffle, + )?; + + for (current, start, end) in segments { + let start = (start * dsz) as usize; + let end = (end * dsz) as usize; + let current = (current * dsz) as usize; + + debug_assert!(start <= end); + debug_assert!(end <= cache.len()); + + dst[current..(current + (end - start))].copy_from_slice(&cache[start..end]); + } + } + } + + Ok(vsz as usize) + } +} + +impl Reader for S3Reader<'_, D> { + fn order(&self) -> Order { + self.ds.order + } + + fn dsize(&self) -> usize { + self.ds.dsize + } + + fn shape(&self) -> &[u64] { + &self.ds.shape + } + + fn read_to(&mut self, extents: &Extents, dst: &mut [u8]) -> Result { + block_on(self.read_to_async(extents, dst)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn chunks() -> Vec> { + // contiguous, gap of 1k, gap > COALESCE_GAP + vec![ + Chunk::new(1000, 500, [0]), + Chunk::new(1500, 500, [10]), + Chunk::new(2500, 500, [20]), + Chunk::new(3000 + 2 * COALESCE_GAP, 500, [30]), + ] + } + + #[test] + fn group_coalesced() { + let chunks = chunks(); + let groups = chunks.iter().map(|c| (c, 0, 0, 10)).collect::>(); + + let requests = group_requests(groups); + assert_eq!(requests.len(), 2); + + // chunk 0 and 1 are contiguous, chunk 2 is within COALESCE_GAP. + assert_eq!(requests[0].start, 1000); + assert_eq!(requests[0].end, 3000); + assert_eq!(requests[0].chunks.len(), 3); + + assert_eq!(requests[1].start, 3000 + 2 * COALESCE_GAP); + assert_eq!(requests[1].chunks.len(), 1); + } + + #[test] + fn group_slices_of_same_chunk() { + let chunks = chunks(); + let groups = vec![ + (&chunks[0], 0, 0, 10), + (&chunks[0], 10, 20, 30), + (&chunks[1], 20, 0, 10), + ]; + + let requests = group_requests(groups); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].chunks.len(), 2); + assert_eq!(requests[0].chunks[0].1, vec![(0, 0, 10), (10, 20, 30)]); + } + + #[test] + fn group_max_request_size() { + let chunks = [ + Chunk::new(1000, MAX_REQUEST_SZ - 500, [0]), + Chunk::new(1000 + MAX_REQUEST_SZ - 500, 1000, [10]), + ]; + let groups = chunks.iter().map(|c| (c, 0, 0, 10)).collect::>(); + + let requests = group_requests(groups); + assert_eq!(requests.len(), 2); + } + + #[test] + fn group_empty() { + let requests = group_requests(Vec::<(&Chunk<1>, u64, u64, u64)>::new()); + assert!(requests.is_empty()); + } +} diff --git a/tests/docker-compose.minio.yml b/tests/docker-compose.minio.yml new file mode 100644 index 0000000..d03df85 --- /dev/null +++ b/tests/docker-compose.minio.yml @@ -0,0 +1,9 @@ +services: + minio: + image: quay.io/minio/minio + command: server /data + ports: + - "9000:9000" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin diff --git a/tests/read_s3.rs b/tests/read_s3.rs new file mode 100644 index 0000000..88c2612 --- /dev/null +++ b/tests/read_s3.rs @@ -0,0 +1,276 @@ +#![cfg(feature = "s3")] +//! Integration tests for the S3 reader against a local Minio instance. +//! +//! Start Minio with: +//! +//! ```sh +//! docker compose -f tests/docker-compose.minio.yml up -d +//! ``` +//! +//! and run the tests with: +//! +//! ```sh +//! HIDEFIX_S3_ENDPOINT=http://localhost:9000 cargo test --features s3 --test read_s3 +//! ``` +//! +//! The tests are skipped when `HIDEFIX_S3_ENDPOINT` is not set. + +use std::sync::{Mutex, OnceLock}; + +use hidefix::idx::DatasetD; +use hidefix::prelude::*; +use hidefix::reader::s3::S3Reader; +use s3::creds::Credentials; +use s3::{Bucket, BucketConfiguration, Region}; + +const BUCKET: &str = "hidefix-test"; + +const FILES: &[&str] = &[ + "tests/data/coads_climatology.nc4", + "tests/data/dmrpp/chunked_oneD.h5", + "tests/data/dmrpp/chunked_gzipped_twoD.h5", + "tests/data/dmrpp/chunked_shuffled_twoD.h5", +]; + +fn rt() -> &'static tokio::runtime::Runtime { + static RT: OnceLock = OnceLock::new(); + RT.get_or_init(|| tokio::runtime::Runtime::new().unwrap()) +} + +fn region() -> Option { + let endpoint = std::env::var("HIDEFIX_S3_ENDPOINT").ok()?; + Some(Region::Custom { + region: "us-east-1".into(), + endpoint, + }) +} + +fn credentials() -> Credentials { + let access = std::env::var("HIDEFIX_S3_ACCESS_KEY").unwrap_or_else(|_| "minioadmin".into()); + let secret = std::env::var("HIDEFIX_S3_SECRET_KEY").unwrap_or_else(|_| "minioadmin".into()); + Credentials::new(Some(&access), Some(&secret), None, None, None).unwrap() +} + +/// Bucket with the test files uploaded, or `None` when no endpoint is configured. +fn bucket() -> Option> { + let Some(region) = region() else { + eprintln!("HIDEFIX_S3_ENDPOINT not set, skipping"); + return None; + }; + + let bucket = Bucket::new(BUCKET, region.clone(), credentials()) + .unwrap() + .with_path_style(); + + static SETUP: Mutex = Mutex::new(false); + let mut done = SETUP.lock().unwrap(); + if !*done { + // On a dedicated thread so that `block_on` also works when the calling test runs + // inside a tokio runtime. + std::thread::scope(|scope| { + scope + .spawn(|| { + rt().block_on(async { + if !bucket.exists().await.unwrap() { + Bucket::create_with_path_style( + BUCKET, + region, + credentials(), + BucketConfiguration::default(), + ) + .await + .unwrap(); + } + + for f in FILES { + bucket + .put_object(f, &std::fs::read(f).unwrap()) + .await + .unwrap(); + } + }) + }) + .join() + .unwrap(); + }); + *done = true; + } + + Some(bucket) +} + +#[test] +fn read_coads_sst() { + let Some(bucket) = bucket() else { return }; + + let i = Index::index("tests/data/coads_climatology.nc4").unwrap(); + let DatasetD::D3(ds) = i.dataset("SST").unwrap() else { + panic!() + }; + let mut r = S3Reader::with_dataset(ds, bucket, "tests/data/coads_climatology.nc4").unwrap(); + + let vs = r.values::(..).unwrap(); + + let h = hdf5::File::open(i.path().unwrap()).unwrap(); + let hvs = h.dataset("SST").unwrap().read_raw::().unwrap(); + + assert_eq!(vs, hvs); +} + +#[test] +fn read_coads_sst_slice() { + let Some(bucket) = bucket() else { return }; + + let i = Index::index("tests/data/coads_climatology.nc4").unwrap(); + let DatasetD::D3(ds) = i.dataset("SST").unwrap() else { + panic!() + }; + let mut r = S3Reader::with_dataset(ds, bucket, "tests/data/coads_climatology.nc4").unwrap(); + + let extents = [3..7, 10..80, 0..90]; + let vs = r.values::(extents.clone()).unwrap(); + + let mut local = i.reader("SST").unwrap(); + let lvs = local.values::(extents).unwrap(); + + assert_eq!(vs, lvs); +} + +#[test] +fn read_chunked_1d() { + let Some(bucket) = bucket() else { return }; + + let i = Index::index("tests/data/dmrpp/chunked_oneD.h5").unwrap(); + let DatasetD::D1(ds) = i.dataset("d_4_chunks").unwrap() else { + panic!() + }; + let mut r = S3Reader::with_dataset(ds, bucket, "tests/data/dmrpp/chunked_oneD.h5").unwrap(); + + let vs = r.values::(..).unwrap(); + + let h = hdf5::File::open(i.path().unwrap()).unwrap(); + let hvs = h.dataset("d_4_chunks").unwrap().read_raw::().unwrap(); + + assert_eq!(vs, hvs); +} + +#[test] +fn read_chunked_gzipped_2d() { + let Some(bucket) = bucket() else { return }; + + let i = Index::index("tests/data/dmrpp/chunked_gzipped_twoD.h5").unwrap(); + let DatasetD::D2(ds) = i.dataset("d_4_gzipped_chunks").unwrap() else { + panic!() + }; + let mut r = + S3Reader::with_dataset(ds, bucket, "tests/data/dmrpp/chunked_gzipped_twoD.h5").unwrap(); + + let vs = r.values::(..).unwrap(); + + let h = hdf5::File::open(i.path().unwrap()).unwrap(); + let hvs = h + .dataset("d_4_gzipped_chunks") + .unwrap() + .read_raw::() + .unwrap(); + + assert_eq!(vs, hvs); +} + +#[test] +fn read_chunked_shuffled_2d() { + let Some(bucket) = bucket() else { return }; + + let i = Index::index("tests/data/dmrpp/chunked_shuffled_twoD.h5").unwrap(); + let DatasetD::D2(ds) = i.dataset("d_4_shuffled_chunks").unwrap() else { + panic!() + }; + let mut r = + S3Reader::with_dataset(ds, bucket, "tests/data/dmrpp/chunked_shuffled_twoD.h5").unwrap(); + + let vs = r.values::(..).unwrap(); + + let h = hdf5::File::open(i.path().unwrap()).unwrap(); + let hvs = h + .dataset("d_4_shuffled_chunks") + .unwrap() + .read_raw::() + .unwrap(); + + assert_eq!(vs, hvs); +} + +/// A selection covering the first and third chunk (in file order), but not the second: +/// the chunks are 10k bytes apart, so the coalesced request fetches the middle chunk and +/// discards it. The selection also starts and ends mid-chunk in the last dimension. +#[test] +fn read_shuffled_2d_coalesced_gap() { + let Some(bucket) = bucket() else { return }; + + let i = Index::index("tests/data/dmrpp/chunked_shuffled_twoD.h5").unwrap(); + let DatasetD::D2(ds) = i.dataset("d_4_shuffled_chunks").unwrap() else { + panic!() + }; + let mut r = + S3Reader::with_dataset(ds, bucket, "tests/data/dmrpp/chunked_shuffled_twoD.h5").unwrap(); + + // Chunks are 50 x 50 in a 100 x 100 dataset: 0..100 x 10..40 hits chunks (0, 0) and + // (50, 0) but not (0, 50), which lies between them in the file. + let vs = r.values::([0..100, 10..40]).unwrap(); + + let h = hdf5::File::open(i.path().unwrap()).unwrap(); + let hvs = h + .dataset("d_4_shuffled_chunks") + .unwrap() + .read_raw::() + .unwrap(); + + let expected = (0..100) + .flat_map(|row| hvs[(row * 100 + 10)..(row * 100 + 40)].iter().copied()) + .collect::>(); + + assert_eq!(vs, expected); +} + +/// The sync `Reader` interface should also work from within a current-thread runtime +/// (`#[tokio::test]` default), where `block_in_place` is not available. +#[tokio::test] +async fn read_chunked_1d_current_thread_runtime() { + let Some(bucket) = bucket() else { return }; + + let i = Index::index("tests/data/dmrpp/chunked_oneD.h5").unwrap(); + let DatasetD::D1(ds) = i.dataset("d_4_chunks").unwrap() else { + panic!() + }; + let mut r = S3Reader::with_dataset(ds, bucket, "tests/data/dmrpp/chunked_oneD.h5").unwrap(); + + let vs = r.values::(..).unwrap(); + + let h = hdf5::File::open(i.path().unwrap()).unwrap(); + let hvs = h.dataset("d_4_chunks").unwrap().read_raw::().unwrap(); + + assert_eq!(vs, hvs); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn read_coads_sst_async() { + let Some(bucket) = bucket() else { return }; + + let i = Index::index("tests/data/coads_climatology.nc4").unwrap(); + let DatasetD::D3(ds) = i.dataset("SST").unwrap() else { + panic!() + }; + let r = S3Reader::with_dataset(ds, bucket, "tests/data/coads_climatology.nc4").unwrap(); + + let h = hdf5::File::open(i.path().unwrap()).unwrap(); + let hvs = h.dataset("SST").unwrap().read_raw::().unwrap(); + + let mut dst = vec![0u8; hvs.len() * 4]; + let sz = r.read_to_async(&(..).into(), &mut dst).await.unwrap(); + assert_eq!(sz, dst.len()); + + // The sync `Reader` interface should also work from within a multi-threaded runtime. + let mut r = r; + let vs = r.values::(..).unwrap(); + assert_eq!(vs, hvs); +}