diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e371be..ef1dea5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,3 +58,148 @@ jobs: - name: Check run: pnpm check + + native-check: + name: Native check (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-15 + - windows-latest + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up pnpm + uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + with: + version: 10.34.5 + run_install: false + + - name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Test native crate + run: cargo test --workspace --locked + + - name: Build host binding + run: pnpm native:build + + - name: Build package + run: pnpm build + + - name: Test native mode auto + run: node scripts/native-mode-smoke.mjs auto + + - name: Test native mode require + run: node scripts/native-mode-smoke.mjs require + + - name: Test native mode off + run: node scripts/native-mode-smoke.mjs off + + - name: Test native security and path equivalence + run: pnpm test test/native-integration.test.ts test/native-archive-equivalence.test.ts test/native-publish-equivalence.test.ts test/private-directory.test.ts + + native-musl: + name: Native check (linux-x64-musl) + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Build and test in Alpine + run: | + docker run --rm -v "$GITHUB_WORKSPACE:/work" -w /work node:24-alpine sh -euxc ' + apk add --no-cache cargo rust musl-dev build-base python3 + npm install --global pnpm@10.34.5 + pnpm install --frozen-lockfile + cargo test --workspace --locked + pnpm native:build + pnpm build + node scripts/native-mode-smoke.mjs auto + node scripts/native-mode-smoke.mjs require + node scripts/native-mode-smoke.mjs off + pnpm test test/native-archive-equivalence.test.ts test/native-publish-equivalence.test.ts + ' + + cargo-quality: + name: Cargo clippy and audit + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Cache Cargo + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: cargo-quality-${{ runner.os }}-${{ hashFiles('Cargo.lock') }} + + - name: Clippy + run: cargo clippy --workspace --locked -- -D warnings + + - name: Install cargo-audit + run: cargo install cargo-audit --version 0.22.2 --locked + + - name: Audit + run: cargo audit --deny warnings + + release-graph-smoke: + name: Release graph smoke (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-15 + - windows-latest + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up pnpm + uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + with: + version: 10.34.5 + run_install: false + + - name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build host binding + run: pnpm native:build + + - name: Pack and smoke test release graph + run: pnpm release-graph:smoke + + - name: Upload behavior proof + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-graph-proof-${{ runner.os }}-${{ runner.arch }} + path: release-graph-proof.json + if-no-files-found: error diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index a0088f0..9ecc6f4 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -37,6 +37,9 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile + - name: Build host binding + run: pnpm native:build + - name: Coverage run: pnpm test:coverage diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a40a2f5..bdddb28 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,17 +14,15 @@ concurrency: env: NODE_VERSION: 24 + NPM_VERSION: 12.0.1 PNPM_VERSION: 10.34.5 jobs: - publish: - name: Publish @openclaw/fs-safe + validate: + name: Validate release source if: github.ref_protected runs-on: ubuntu-latest - timeout-minutes: 20 - permissions: - contents: read - id-token: write + timeout-minutes: 25 steps: - name: Check out uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -47,174 +45,297 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - - name: Validate package metadata - run: | - node <<'NODE' - const { readFileSync } = require("node:fs"); - - const pkg = JSON.parse(readFileSync("package.json", "utf8")); - const expectedName = "@openclaw/fs-safe"; - const expectedAuthor = "OpenClaw Team "; - const expectedRepo = "https://github.com/openclaw/fs-safe"; - const repository = typeof pkg.repository === "string" - ? pkg.repository - : pkg.repository?.url; - const normalize = (value) => - String(value ?? "") - .trim() - .replace(/^git\+/, "") - .replace(/\.git$/i, "") - .replace(/\/+$/, ""); - - const errors = []; - if (pkg.name !== expectedName) { - errors.push(`package name must be ${expectedName}; found ${pkg.name ?? ""}`); - } - if (pkg.author !== expectedAuthor) { - errors.push(`package author must be ${expectedAuthor}; found ${pkg.author ?? ""}`); - } - if (normalize(repository) !== expectedRepo) { - errors.push(`repository URL must be ${expectedRepo}; found ${normalize(repository) || ""}`); - } - if (pkg.private === true) { - errors.push("package must not be private"); - } - if (pkg.publishConfig?.access !== "public") { - errors.push("publishConfig.access must be public"); - } - if (pkg.publishConfig?.provenance !== true) { - errors.push("publishConfig.provenance must be true"); - } - if (errors.length > 0) { - for (const error of errors) console.error(error); - process.exit(1); - } - NODE - - - name: Validate release tag and changelog + - name: Validate tag, changelog, and package versions env: RELEASE_TAG: ${{ github.ref_name }} run: | set -euo pipefail git fetch --no-tags origin main - - package_version="$(node -p "require('./package.json').version")" - expected_tag="v${package_version}" if ! printf '%s\n' "${RELEASE_TAG}" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then echo "release tag must match vX.Y.Z; received ${RELEASE_TAG}" >&2 exit 1 fi - if [ "${RELEASE_TAG}" != "${expected_tag}" ]; then - echo "release tag ${RELEASE_TAG} does not match package version ${package_version}" >&2 - exit 1 - fi if [ "$(git cat-file -t "refs/tags/${RELEASE_TAG}")" != "tag" ]; then echo "release tag ${RELEASE_TAG} must be annotated" >&2 exit 1 fi release_commit="$(git rev-parse "refs/tags/${RELEASE_TAG}^{commit}")" git merge-base --is-ancestor "${release_commit}" origin/main + node <<'NODE' + const { readFileSync } = require("node:fs"); + const { join } = require("node:path"); + const version = process.env.RELEASE_TAG.slice(1); + const dirs = [ + ".", "native", "npm/linux-x64-gnu", "npm/linux-x64-musl", + "npm/linux-arm64-gnu", "npm/linux-arm64-musl", "npm/darwin-x64", + "npm/darwin-arm64", "npm/win32-x64-msvc", + ]; + for (const dir of dirs) { + const pkg = JSON.parse(readFileSync(join(dir, "package.json"), "utf8")); + if (pkg.version !== version) throw new Error(`${pkg.name} is ${pkg.version}, expected ${version}`); + } + const cargo = readFileSync("native/Cargo.toml", "utf8"); + if (!cargo.match(new RegExp(`^version = "${version.replaceAll(".", "\\.")}"$`, "m"))) { + throw new Error(`native/Cargo.toml does not declare ${version}`); + } + const loader = readFileSync("native/index.js", "utf8"); + if (!loader.includes("require('./package.json').version")) { + throw new Error("native loader must derive its version check from package.json"); + } + NODE + package_version="$(node -p "require('./package.json').version")" + [ "${RELEASE_TAG}" = "v${package_version}" ] node scripts/release-notes.mjs "${package_version}" >/dev/null - - name: Check + - name: Check TypeScript package run: pnpm check - - name: Prepare release artifact - id: artifact - run: | - set -euo pipefail - mkdir release-artifact - npm pack \ - --json \ - --ignore-scripts \ - --pack-destination release-artifact \ - > release-artifact/pack.json - node <<'NODE' >> "${GITHUB_OUTPUT}" - const { readFileSync } = require("node:fs"); - const [artifact] = JSON.parse(readFileSync("release-artifact/pack.json", "utf8")); + - name: Test native crate + run: cargo test --workspace --locked - if (!artifact?.filename || !artifact?.integrity) { - console.error("npm pack did not return an artifact filename and integrity"); - process.exit(1); - } - console.log(`filename=${artifact.filename}`); - console.log(`integrity=${artifact.integrity}`); - NODE + prebuild: + name: Build ${{ matrix.label }} + if: github.ref_protected + needs: validate + runs-on: ${{ matrix.os }} + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + include: + - label: linux-x64-gnu + os: ubuntu-latest + target: x86_64-unknown-linux-gnu + cross: none + - label: linux-x64-musl + os: ubuntu-latest + target: x86_64-unknown-linux-musl + cross: zig + - label: linux-arm64-gnu + os: ubuntu-latest + target: aarch64-unknown-linux-gnu + cross: napi + - label: linux-arm64-musl + os: ubuntu-latest + target: aarch64-unknown-linux-musl + cross: zig + - label: darwin-x64 + os: macos-15 + target: x86_64-apple-darwin + cross: none + - label: darwin-arm64 + os: macos-15 + target: aarch64-apple-darwin + cross: none + - label: win32-x64-msvc + os: windows-latest + target: x86_64-pc-windows-msvc + cross: none + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Publish or verify npm artifact - env: - PACKAGE_FILENAME: ${{ steps.artifact.outputs.filename }} - PACKAGE_INTEGRITY: ${{ steps.artifact.outputs.integrity }} - run: | - set -euo pipefail - package_name="$(node -p "require('./package.json').name")" - package_version="$(node -p "require('./package.json').version")" - package_spec="${package_name}@${package_version}" - - verify_registry_artifact() { - registry_integrity="$(npm view "${package_spec}" dist.integrity 2>registry-error.log)" - view_status=$? - if [ "${view_status}" -ne 0 ]; then - if grep -q "E404" registry-error.log; then - return 2 - fi - cat registry-error.log >&2 - return 3 - fi - registry_attestation="$(npm view "${package_spec}" dist.attestations.url)" - if [ "${registry_integrity}" != "${PACKAGE_INTEGRITY}" ]; then - echo "${package_spec} exists with different package bytes." >&2 - return 1 - fi - if [ -z "${registry_attestation}" ]; then - echo "${package_spec} exists without npm provenance." >&2 - return 1 - fi - echo "verified ${package_spec} integrity and provenance" - } + - name: Set up pnpm + uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + with: + version: ${{ env.PNPM_VERSION }} + run_install: false + + - name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ env.NODE_VERSION }} + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + + - name: Set up Zig + if: matrix.cross == 'zig' + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2 + with: + version: 0.15.2 + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Add Rust target + run: rustup target add ${{ matrix.target }} + + - name: Install cargo-zigbuild + if: matrix.cross == 'zig' + run: cargo install cargo-zigbuild --version 0.23.0 --locked + + - name: Build native binding + if: matrix.cross == 'none' + run: pnpm --filter @openclaw/fs-safe-native exec napi build --release --platform --target ${{ matrix.target }} --output-dir ../artifacts + + - name: Build native binding with napi cross toolchain + if: matrix.cross == 'napi' + run: pnpm --filter @openclaw/fs-safe-native exec napi build --release --platform --target ${{ matrix.target }} --use-napi-cross --output-dir ../artifacts + + - name: Build native binding with Zig + if: matrix.cross == 'zig' + run: pnpm --filter @openclaw/fs-safe-native exec napi build --release --platform --target ${{ matrix.target }} --cross-compile --output-dir ../artifacts + + - name: Upload binding + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: binding-${{ matrix.label }} + path: artifacts/*.node + if-no-files-found: error + retention-days: 1 + + collect: + name: Validate release packages + if: github.ref_protected + needs: prebuild + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up pnpm + uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + with: + version: ${{ env.PNPM_VERSION }} + run_install: false + + - name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ env.NODE_VERSION }} + cache: pnpm + cache-dependency-path: pnpm-lock.yaml + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Download bindings + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: binding-* + path: artifacts + merge-multiple: true + + - name: Assemble platform packages + run: pnpm --filter @openclaw/fs-safe-native exec napi artifacts --output-dir ../artifacts --npm-dir ../npm + + - name: Build root package + run: pnpm build - set +e - verify_registry_artifact - verify_status=$? - set -e - case "${verify_status}" in - 0) exit 0 ;; - 2) ;; - *) exit "${verify_status}" ;; - esac - - set +e - npm publish \ - "./release-artifact/${PACKAGE_FILENAME}" \ - --access public \ - --provenance - publish_status=$? - set -e - - for attempt in $(seq 1 12); do - set +e - verify_registry_artifact - verify_status=$? - set -e - if [ "${verify_status}" -eq 0 ]; then - exit 0 - fi - if [ "${verify_status}" -eq 3 ]; then - exit "${verify_status}" - fi - echo "registry verification attempt ${attempt}/12 did not confirm the artifact yet" - sleep 5 - done - - echo "npm publish exited ${publish_status}; registry verification never confirmed the artifact." >&2 - exit 1 + - name: Pack and smoke test every package + run: node scripts/check-release-packages.mjs --output release-artifacts + + - name: Upload release packages + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: release-packages + path: release-artifacts/* + if-no-files-found: error + retention-days: 7 + + publish-platforms: + name: Publish ${{ matrix.package }} + if: github.ref_protected + needs: collect + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + id-token: write + strategy: + fail-fast: false + matrix: + package: + - "@openclaw/fs-safe-native-linux-x64-gnu" + - "@openclaw/fs-safe-native-linux-x64-musl" + - "@openclaw/fs-safe-native-linux-arm64-gnu" + - "@openclaw/fs-safe-native-linux-arm64-musl" + - "@openclaw/fs-safe-native-darwin-x64" + - "@openclaw/fs-safe-native-darwin-arm64" + - "@openclaw/fs-safe-native-win32-x64-msvc" + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ env.NODE_VERSION }} + registry-url: https://registry.npmjs.org + + - name: Update npm + run: npm install --global npm@${{ env.NPM_VERSION }} + + - name: Download release packages + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: release-packages + path: release-artifacts + + - name: Publish or verify + run: node scripts/publish-or-verify.mjs --package "${{ matrix.package }}" --artifacts release-artifacts + + publish-native: + name: Publish @openclaw/fs-safe-native + if: github.ref_protected + needs: publish-platforms + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + id-token: write + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ env.NODE_VERSION }} + registry-url: https://registry.npmjs.org + - name: Update npm + run: npm install --global npm@${{ env.NPM_VERSION }} + - name: Download release packages + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: release-packages + path: release-artifacts + - name: Publish or verify + run: node scripts/publish-or-verify.mjs --package "@openclaw/fs-safe-native" --artifacts release-artifacts + + publish-root: + name: Publish @openclaw/fs-safe + if: github.ref_protected + needs: publish-native + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + id-token: write + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ env.NODE_VERSION }} + registry-url: https://registry.npmjs.org + - name: Update npm + run: npm install --global npm@${{ env.NPM_VERSION }} + - name: Download release packages + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: release-packages + path: release-artifacts + - name: Publish or verify + run: node scripts/publish-or-verify.mjs --package "@openclaw/fs-safe" --artifacts release-artifacts release: name: Publish GitHub Release if: github.ref_protected + needs: publish-root runs-on: ubuntu-latest - timeout-minutes: 5 - needs: publish + timeout-minutes: 10 permissions: contents: write steps: @@ -223,12 +344,24 @@ jobs: with: fetch-depth: 0 - - name: Generate release notes + - name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: Download release packages + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: release-packages + path: release-artifacts + + - name: Generate release notes and proof env: RELEASE_TAG: ${{ github.ref_name }} run: | package_version="${RELEASE_TAG#v}" node scripts/release-notes.mjs "${package_version}" > release-notes.md + node scripts/append-release-proof.mjs release-notes.md release-artifacts/manifest.json "${GITHUB_REPOSITORY}" "${GITHUB_RUN_ID}" - name: Publish release env: diff --git a/.gitignore b/.gitignore index 7dc6520..c8078c0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ dist/ node_modules +target/ +native/*.node coverage/ +release-graph-proof*.json *.tsbuildinfo .clawpatch/ .deepsec/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 1339604..a7cffbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,41 @@ # Changelog -## 0.4.8 - Unreleased +## 0.5.0 - Unreleased + +### Highlights + +- Add policy-driven archive entry filtering and mode handling, bounded single-entry archive reads, root-bounded async walking, synchronous sidecar locks, async secret reads, create-only secret writes, and exclusive file publication. +- Add public streaming `sha256File(path | FileHandle)` hashing with optional async native acceleration, plus policy-free Windows owner/DACL facts with owner and current-process-user SIDs and per-ACE masks and inheritance flags. +- Add native fd-relative ZIP and TAR extraction/read support with gzip, zstd, and bzip2 streaming; TypeScript evaluates the shared entry policy before Rust creates any output, and zstd/bzip2 report a typed native-required error when no binding is available. +- Bound PAX, GNU long-name/link, and sparse metadata with one `maxMetaEntryBytes` policy shared by node-tar and the native fixed-header metering reader, including typed failures for oversized or malformed metadata. +- Add `Root.walk()` subtree pruning and partial directory-error reporting for bounded best-effort consumers, plus a shared `maxEntryPathComponents` archive limit that rejects implicit-directory depth attacks before either extraction path creates output. +- Ship the optional `@openclaw/fs-safe-native` helper and seven platform packages from this repository for fd-relative opens, guarded directory creation and hardlinks, atomic no-replace rename, and file identity checks. + +### Security and Correctness + +- Enforce `movePathWithCopyFallback({ sourceHardlinks: "reject" })` with a streaming, entry-capped recursive preflight before mutation, closing a shipped 0.4.x gap where the common same-filesystem rename bypassed the policy; approved trees commit through a fresh staged copy with open-time and post-copy link-count fences so a scan/rename race cannot publish a hardlinked inode. +- Abort and tear down JavaScript TAR extraction immediately when entry policy, path validation, link rejection, or a budget fails, preventing node-tar from leaving a paused parser after rejected fleet-restore entries; both native and JavaScript paths now return the same typed archive-policy errors. +- Attach a post-creation failure receipt to `publishFileExclusive()` errors with the failing phase, whether this call created the target, its observed identity, and whether cleanup removed, preserved, or could not classify the target. +- Add `publishFileExclusive({ onSyncFailure: "rollback" | "preserve" })`: rollback remains the default, while preserve keeps a complete target after directory-sync failure and reports the failed sync outcome in the typed provenance receipt. +- Close the pinned publication source on parent-pinning failure so every acquired descriptor is released on every exit path. +- Remove the native loader's PATH-resolved `ldd` execution. Linux libc detection now uses the Node process report, conventional musl library filenames, and the Node executable's ELF interpreter without spawning a process at import time; an inconclusive probe conservatively attempts glibc and falls back normally in `auto` mode. +- Default archive extraction to `entryModes: "clamp"`, normalizing directories to `0o755` and files to `0o644` or `0o755` while always stripping setuid, setgid, and sticky bits; use `"preserve"` to retain safe archived rwx bits. +- Prefer native create-only commits, sidecar acquisition, hardlink publication, and the explicit `rename-noreplace` publication strategy when the platform binding is available, while retaining guarded JavaScript fallbacks for `auto` and `off` modes. +- Accelerate exclusive publication fallbacks with macOS `fclonefileat`, Linux `FICLONE` and `copy_file_range`, then the unchanged JavaScript byte loop; all paths retain exclusive creation, identity fencing, mode normalization, and SHA-256 verification through an async native hash task when available. +- Add direct Windows owner/DACL inspection and protected private-directory creation for the current owner, LocalSystem, and Administrators, while retaining the existing .NET/`icacls` behavior when native mode is unavailable, forced off, or encounters an unsupported descriptor form. +- Keep the public private-directory creator Windows-only and native-only so POSIX pathname races or inherited ACLs cannot weaken its privacy guarantee. +- Build Linux native opens on `openat2(RESOLVE_BENEATH | RESOLVE_NO_MAGICLINKS)`, macOS opens on an in-root `O_NOFOLLOW` component walk, and Windows opens on handle-relative `NtCreateFile` with reparse-point rejection. + +### Compatibility + +- Remove the persistent Python helper and its `pythonPath` configuration. Replace `configureFsSafePython`, `FS_SAFE_PYTHON_MODE`, and the OpenClaw Python aliases with `configureFsSafeNative` and `FS_SAFE_NATIVE_MODE`; 0.5 warns once and maps the former `auto`, `require`, and `off` policies solely as an upgrade bridge for shipped 0.4 consumers. +- Add `publishFileExclusive({ strategy: "rename-noreplace" })`; this strategy requires the native helper, atomically moves the source, and never replaces an existing destination. + +### Docs and Tooling + +- Add an ordered 0.4-to-0.5 migration checklist and reconcile every new archive, native, publication, walk, lock, secret, permission, and temp-workspace contract with realistic examples and cross-links. +- Convert the repository to a pnpm workspace, test the Rust crate on Linux, macOS, and Windows, and publish all platform bindings, the native loader, and the root package through one protected-tag release pipeline with npm provenance. +- Replace unused napi-rs Android, FreeBSD, OpenHarmony, WASI, and unsupported-architecture loader branches with a checked-in loader for the seven packages actually published, and make publication benchmarks report the exercised clone/copy/JavaScript tier plus filesystem environment. ## 0.4.7 - 2026-07-24 diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..89e352c --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,616 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "cc" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "convert_case" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctor" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e30e509674ef0ec91e21a7735766db37d163d46151b6a361d8b83dd79116bd" + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fs-safe-native" +version = "0.5.0-dev.0" +dependencies = [ + "bzip2", + "flate2", + "libc", + "napi", + "napi-build", + "napi-derive", + "rustix", + "sha2", + "tar", + "windows-sys", + "zip", + "zstd", +] + +[[package]] +name = "futures" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-executor" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" + +[[package]] +name = "futures-macro" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hybrid-array" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +dependencies = [ + "typenum", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "libbz2-rs-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "napi" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de33522036981030a75c231829566bc63414e08101a6f5ff4ac6cef19c8e0941" +dependencies = [ + "bitflags", + "ctor", + "futures", + "napi-build", + "napi-sys", + "nohash-hasher", + "rustc-hash", +] + +[[package]] +name = "napi-build" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9c366d2c8c60b86fa632df75f745509b52f9128f91a6bad4c796e44abb505e1" + +[[package]] +name = "napi-derive" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a49c513341a61a16a10af6efcce46b30d0822ba2d4fb197d24d33dfc199c78d5" +dependencies = [ + "convert_case", + "ctor", + "napi-derive-backend", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "napi-derive-backend" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4747005fa3e2c9989ac45a723a514c5db2411238b72981a3cda4c701a9dfea17" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "semver", + "syn", +] + +[[package]] +name = "napi-sys" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85fbf1fa9f1babfe396d74bbbf52b3643770243e8f5b0b46715d4caf7f0dfc9a" +dependencies = [ + "libloading", +] + +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zip" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" +dependencies = [ + "arbitrary", + "crc32fast", + "flate2", + "indexmap", + "memchr", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..2180d48 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,10 @@ +[workspace] +members = ["native"] +resolver = "2" + +[profile.release] +lto = true +# Cargo's Mach-O symbol stripping can corrupt the link-edit string table after +# static compression libraries are linked. Platform package assembly may strip +# with the target toolchain after the loadable artifact has been validated. +strip = false diff --git a/README.md b/README.md index 2c9f057..25af2b3 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Full docs and reference at **[fs-safe.io](https://fs-safe.io)**. ## Contents -[Why this exists](#why-this-exists) · [Not a sandbox](#not-a-sandbox) · [Install](#install) · [Quick start](#quick-start) · [Reading](#reading) · [Subpaths](#subpaths) · [Failure semantics](#failure-semantics-in-the-name) · [Directory durability](#directory-durability) · [Atomic writes](#atomic-writes) · [External outputs](#external-outputs) · [Stores](#stores) · [Secure absolute reads](#secure-absolute-file-reads) · [Walking](#directory-walking) · [Archive extraction](#archive-extraction) · [Path scopes](#advanced-path-scopes) · [Errors](#errors) · [Safety model](#safety-model) · [Limitations](#limitations) +[Why this exists](#why-this-exists) · [Not a sandbox](#not-a-sandbox) · [Install](#install) · [0.5 migration](docs/migrating-to-0.5.md) · [Python migration](#migrating-from-the-python-helper) · [Quick start](#quick-start) · [Reading](#reading) · [Subpaths](#subpaths) · [Failure semantics](#failure-semantics-in-the-name) · [Directory durability](#directory-durability) · [Atomic writes](#atomic-writes) · [External outputs](#external-outputs) · [Stores](#stores) · [Secure absolute reads](#secure-absolute-file-reads) · [Walking](#directory-walking) · [Archive extraction](#archive-extraction) · [Path scopes](#advanced-path-scopes) · [Errors](#errors) · [Safety model](#safety-model) · [Limitations](#limitations) ## Why this exists @@ -45,7 +45,7 @@ The same idea has landed in other languages. Go [added `os.Root` and `OpenInRoot | `path.resolve().startsWith()` | string check only | – | – | – | – | | [`write-file-atomic`](https://www.npmjs.com/package/write-file-atomic) | – | ✓ | – | – | – | | Go [`os.Root`](https://go.dev/blog/osroot) / Rust [`cap-std`](https://github.com/bytecodealliance/cap-std) | ✓ | platform | ✓ | ✓ | – | -| **`@openclaw/fs-safe`** | **✓** | **✓** | **✓** | **✓ (POSIX fd-relative)** | **✓ (ZIP/TAR)** | +| **`@openclaw/fs-safe`** | **✓** | **✓** | **✓** | **✓ (POSIX fd-relative)** | **✓ (ZIP/TAR; native zstd/bzip2)** | ## Not a sandbox @@ -59,27 +59,41 @@ pnpm add @openclaw/fs-safe Node 22 or newer. Core root/path/json/temp helpers avoid framework dependencies. Archive helpers use optional `jszip` and `tar` dependencies for ZIP/TAR support; installs that omit optional dependencies can still use every non-archive subpath. -On POSIX, `root()` uses one process-global persistent Python helper for the -fd-relative operations Node does not expose ergonomically (`renameat`, -`unlinkat`, recursive `mkdirat`-style walks, and parent-fd writes). Configure it -before first use when you need a strict environment policy: +The optional native package supplies fd-relative and atomic no-replace +primitives that Node does not expose directly. Configure its lazy loader before +first use when you need a strict environment policy: ```ts -import { configureFsSafePython } from "@openclaw/fs-safe"; +import { configureFsSafeNative } from "@openclaw/fs-safe"; -configureFsSafePython({ mode: "auto" }); // default: use helper, fall back if unavailable -configureFsSafePython({ mode: "off" }); // never spawn Python; use best-effort Node fallbacks -configureFsSafePython({ mode: "require" }); // fail closed if helper cannot start +configureFsSafeNative({ mode: "auto" }); // default: native when available +configureFsSafeNative({ mode: "off" }); // guarded JavaScript only +configureFsSafeNative({ mode: "require" }); // fail closed if the binding is unavailable ``` -Equivalent env vars: `FS_SAFE_PYTHON_MODE=auto|off|require` and -`FS_SAFE_PYTHON=/path/to/python3`. Without Python, `fs-safe` keeps lexical and -canonical root checks, no-follow opens, atomic temp+rename writes, and -post-write identity verification. What you lose is the strongest POSIX -fd-relative protection against a same-process-user racer swapping parent -directories between validation and mutation. Windows already uses the Node -fallback path. See the [Python helper policy](docs/python-helper.md) for -deployment guidance. +Equivalent env var: `FS_SAFE_NATIVE_MODE=auto|off|require`. The seven platform +packages are prebuilt; consumers do not need Rust. Without a matching package, +`auto` retains lexical and canonical root checks, no-follow opens, guarded +temp+rename writes, and post-write identity verification. See the [native +helper policy](docs/native-helper.md) for the exact boundary and deployment +tradeoff, and [native architecture](docs/native.md) for the platform mechanisms +and policy ownership model. + +## Migrating from the Python helper + +Version 0.5 replaces the persistent Python worker with optional prebuilt native +bindings. The modes map directly: `configureFsSafePython({ mode: "auto" })` +becomes `configureFsSafeNative({ mode: "auto" })`, and likewise for `off` and +`require`. Replace `FS_SAFE_PYTHON_MODE` with `FS_SAFE_NATIVE_MODE`; remove +`pythonPath`, `FS_SAFE_PYTHON`, and interpreter provisioning because the native +loader does not spawn Python. + +Version 0.5 retains the old function and documented `FS_SAFE_PYTHON*` +and OpenClaw Python environment names emit one `FS_SAFE_PYTHON_DEPRECATED` +warning and map the old mode to its native equivalent. They are migration +bridges for shipped 0.4 consumers, not an alternate helper contract. Update +startup configuration as part of the 0.5 upgrade rather than relying on the +warning path. Follow the [0.5 migration checklist](docs/migrating-to-0.5.md). ## Quick start @@ -174,32 +188,32 @@ await locked.write(".env", "token"); // FsSafeError code "denied-path" ## Subpaths -The main entry point is intentionally small: `root`, the root option/result -types, and `FsSafeError`. Use subpaths for everything else. Low-level helpers -that OpenClaw needs to compose higher-level APIs are grouped under +The main entry point collects the common root, config, output, lock, native-mode, +and error exports. Prefer focused subpaths when a consumer needs a narrower +contract. Low-level helpers that OpenClaw needs to compose higher-level APIs are grouped under `@openclaw/fs-safe/advanced` instead of being separate public leaf contracts. | Subpath | Contents | |---|---| -| `@openclaw/fs-safe/root` | `root()`, `Root`, `RootDefaults`, related types | -| `@openclaw/fs-safe/config` | process-global Python helper configuration | +| `@openclaw/fs-safe/root` | `root()`, `Root`, `RootDefaults`, and root-bounded walking with pruning/error markers | +| `@openclaw/fs-safe/config` | process-global native helper and lock defaults | | `@openclaw/fs-safe/path` | canonical path checks: `isPathInside`, `safeRealpathSync`, `isNotFoundPathError`, `isSymlinkOpenError` | | `@openclaw/fs-safe/json` | `tryReadJson`, `readJson`, `readJsonIfExists`, `writeJson`, sync variants | | `@openclaw/fs-safe/output` | `writeExternalFileWithinRoot` for external libraries that need a temp output path | | `@openclaw/fs-safe/store` | `fileStore`, `fileStoreSync`, and `jsonStore` | -| `@openclaw/fs-safe/secret` | strict and try-style secret file read/write helpers | +| `@openclaw/fs-safe/secret` | sync/async strict and try-style secret reads, atomic replace, and create-only secret writes | | `@openclaw/fs-safe/atomic` | `replaceFileAtomic`, `replaceFileAtomicSync`, `replaceDirectoryAtomic`, `movePathWithCopyFallback` | -| `@openclaw/fs-safe/durability` | pinned directory identities, strict and best-effort directory sync, durable nested-directory creation | +| `@openclaw/fs-safe/durability` | pinned directory identities, strict directory sync, durable nested-directory creation, exclusive publication, streaming SHA-256, provenance receipts, and sync-failure policy | | `@openclaw/fs-safe/temp` | `tempWorkspace`, `tempWorkspaceSync`, `withTempWorkspace`, `resolveSecureTempRoot` | | `@openclaw/fs-safe/secure-file` | fd-pinned absolute file reads with owner, mode, ACL, trusted-dir, size, and timeout checks | -| `@openclaw/fs-safe/file-lock` | `acquireFileLock`, `withFileLock`, `createFileLockManager`, and related lock types | -| `@openclaw/fs-safe/permissions` | POSIX mode and Windows ACL inspection plus remediation formatting helpers | +| `@openclaw/fs-safe/file-lock` | async/sync sidecar locks, root-bounded sidecars, ownership verification, and stale policy | +| `@openclaw/fs-safe/permissions` | POSIX mode and Windows ACL inspection, raw owner/ACE facts, private-directory creation, and remediation helpers | | `@openclaw/fs-safe/walk` | budget-bounded directory walking with symlink policy, filters, and truncation accounting; not root-bounded | -| `@openclaw/fs-safe/archive` | `extractArchive`, `resolveArchiveKind`, `ArchiveLimitError`, preflight helpers | +| `@openclaw/fs-safe/archive` | policy-driven ZIP/TAR extraction, clamp/filter policy, metadata/path-depth limits, native gzip/zstd/bzip2, and bounded entry reads | | `@openclaw/fs-safe/advanced` | lower-level composition helpers such as path scopes, root-file open, bounded descriptor reads, install paths, filename sanitizing, temp-file targets, sibling-temp writes, local-root readers, regular-file helpers, `pathExists`, and `withTimeout`; less stable than focused public subpaths | -| `@openclaw/fs-safe/errors` | `FsSafeError`, `FsSafeErrorCode` | +| `@openclaw/fs-safe/errors` | `FsSafeError`, closed codes/categories, causes, and operation-specific details receipts | | `@openclaw/fs-safe/types` | shared types: `DirEntry`, `PathStat`, … | -| `@openclaw/fs-safe/test-hooks` | hooks the test suite uses to inject races; only active under `NODE_ENV=test` | +| `@openclaw/fs-safe/test-hooks` | hooks the test suite uses to inject races; registration requires `NODE_ENV=test` or `VITEST=true` | ## Failure semantics in the name @@ -243,8 +257,14 @@ when creating a nested directory. Strict sync propagates real I/O failures and reports known Windows directory-flush limitations explicitly. Separate best-effort helpers preserve operations that do not promise crash durability. +`publishFileExclusive()` adds no-clobber hardlink/copy/rename strategies and a +typed post-creation receipt. Its `onSyncFailure` policy defaults to +`"rollback"`; backup writers can choose `"preserve"` to keep a complete target +when parent-directory sync fails, then inspect `details.directorySync` and +retry or record the weaker durability state. + See [Directory durability](docs/durability.md) for the receipt, pin lifecycle, -creation callback, and platform contract. +publication policy, creation callback, and platform contract. ## Atomic writes @@ -406,6 +426,13 @@ for (const file of scan.entries) { Check `scan.truncated` before treating the result as complete, and `scan.failedDirs` to tell an incomplete scan (a directory that could not be read) from an empty one before pruning state from the listing. +For caller-controlled paths, `Root.walk()` is the root-bounded async iterator. +It supports entry/depth budgets, in-root symlink following, cancellation, and a +truncation marker (or typed error) when a budget is reached. Its `entryFilter` +can return `"skip-subtree"` to prune a directory, and +`onDirectoryError: "skip-and-report"` yields typed `"directory-error"` markers +while preserving entries from readable subtrees. + ## Archive extraction `extractArchive()` handles ZIP and TAR behind one API, with traversal checks, blocked-link-type rejection, and entry-count and byte budgets. @@ -426,6 +453,7 @@ await extractArchive({ maxEntries: 50_000, maxExtractedBytes: 512 * 1024 * 1024, maxEntryBytes: 256 * 1024 * 1024, + maxEntryPathComponents: 64, }, }); ``` @@ -474,19 +502,19 @@ if (err instanceof FsSafeError) { } ``` -Current `FsSafeErrorCode` values are `already-exists`, `denied-path`, `device-path`, `hardlink`, `helper-failed`, `helper-unavailable`, `invalid-path`, `insecure-permissions`, `not-empty`, `not-file`, `not-found`, `not-owned`, `not-removable`, `outside-workspace`, `path-alias`, `path-mismatch`, `permission-unverified`, `symlink`, `timeout`, `too-large`, and `unsupported-platform`. +Current `FsSafeErrorCode` values are `already-exists`, `denied-path`, `device-path`, `hardlink`, `helper-failed`, `helper-unavailable`, `invalid-path`, `insecure-permissions`, `not-empty`, `not-file`, `not-found`, `not-owned`, `not-removable`, `outside-workspace`, `path-alias`, `path-mismatch`, `permission-unverified`, `secret-exists`, `symlink`, `timeout`, `too-large`, and `unsupported-platform`. ## Safety model - root-bounded APIs resolve paths against a configured root and reject canonical escapes - reads reject known unsafe device paths, open with `O_NOFOLLOW` where available, then verify fd identity matches the path identity before returning the buffer or handle -- writes use pinned parent-directory helpers and atomic replacement on POSIX, with verified post-write identity -- `remove`, `mkdir`, `move`, `stat`, `list`, and parent-fd writes use one persistent fd-relative Python helper on POSIX, with Node fallbacks when the helper is disabled or unavailable +- create-only writes, sidecar acquisition, and exclusive publication prefer fd-relative native primitives, with verified guarded JavaScript fallbacks +- `remove`, `mkdir`, `move`, `stat`, and `list` retain guarded JavaScript implementations with pre/post identity checks - archive extraction stages into a private directory and merges through the same boundary checks used by direct writes ## Limitations -- Windows uses the safest Node-level behavior available; some fd-relative POSIX hardening is unavailable there. +- Windows native opens are handle-relative and reject reparse points; operations without native wiring use the guarded Node implementation. - Hardlink rejection depends on platform metadata. Treat it as defense-in-depth, not authorization. - `fs-safe` does not validate file contents or archive payload semantics beyond filesystem safety constraints. Schemas, signatures, and authorization belong in the layer above. diff --git a/RELEASE-PREREQS.md b/RELEASE-PREREQS.md new file mode 100644 index 0000000..d280e27 --- /dev/null +++ b/RELEASE-PREREQS.md @@ -0,0 +1,39 @@ +# 0.5.0 release prerequisites + +Complete this checklist in npm before pushing the first `v0.5.0` tag. The release workflow uses npm trusted publishing and has no npm token fallback. + +## Package setup + +Create or confirm these public packages under the `@openclaw` scope: + +- `@openclaw/fs-safe-native-linux-x64-gnu` +- `@openclaw/fs-safe-native-linux-x64-musl` +- `@openclaw/fs-safe-native-linux-arm64-gnu` +- `@openclaw/fs-safe-native-linux-arm64-musl` +- `@openclaw/fs-safe-native-darwin-x64` +- `@openclaw/fs-safe-native-darwin-arm64` +- `@openclaw/fs-safe-native-win32-x64-msvc` +- `@openclaw/fs-safe-native` +- `@openclaw/fs-safe` + +For every package above, open npm package settings and configure the same trusted publisher: + +| npm trusted-publisher field | Required value | +|---|---| +| Provider | GitHub Actions | +| Organization or user | `openclaw` | +| Repository | `fs-safe` | +| Workflow filename | `release.yml` | +| Environment | Leave empty; this workflow does not use a GitHub environment. | + +Confirm each package is public, the OpenClaw npm organization owns it, and the publishing maintainers have 2FA enabled. Do not add `NPM_TOKEN` or an automation token to GitHub. + +## Release commit and tag + +- Replace `0.5.0-dev.0` with `0.5.0` in the root, native loader, Rust crate, all seven platform package manifests, and generated native loader version checks. +- Run `pnpm install` so `pnpm-lock.yaml` matches the stable package versions. +- Change `## 0.5.0 - Unreleased` in `CHANGELOG.md` to the release date. +- Run `pnpm check`, `pnpm test:security`, `cargo test --workspace --locked`, `pnpm docs:site`, and `git diff --check` on the release commit. +- Merge the release commit to `main`, then create an annotated, protected `v0.5.0` tag on that exact commit. + +The workflow validates the protected annotated tag, `main` ancestry, all nine package versions, package bytes, install/import behavior, and changelog-derived release notes before it publishes anything. diff --git a/docs/archive.md b/docs/archive.md index 8bcc532..875012d 100644 --- a/docs/archive.md +++ b/docs/archive.md @@ -1,12 +1,11 @@ # Archive extraction -`@openclaw/fs-safe/archive` extracts ZIP and TAR archives behind one API, with traversal checks, blocked-link-type rejection, and entry-count and byte budgets. Extraction stages into a private directory and merges through the same safe-open boundary used by direct writes — a symlinked entry can't trick the merge into following an out-of-tree path. +`@openclaw/fs-safe/archive` extracts ZIP and TAR archives behind one API, with traversal checks, blocked-link-type rejection, and entry-count and byte budgets. When the optional native binding is available, Rust streams ZIP, TAR, gzip, zstd, and bzip2 while TypeScript remains the sole policy owner; every accepted output is created fd-relative in a private staging root. Extraction then merges through the same safe-open boundary used by direct writes — a symlinked entry can't trick the merge into following an out-of-tree path. -Archive extraction uses optional runtime dependencies: `jszip` for ZIP and `tar` -for TAR. Installs that omit optional dependencies can still import this subpath, -inspect archive kinds, and use pure path/limit helpers, but extraction or ZIP -loading fails with a clear message until the matching optional dependency is -installed. +The guarded JavaScript fallback uses optional runtime dependencies: `jszip` for +ZIP and `tar` for TAR/gzip. The native path does not use those packages. Installs +that omit optional dependencies can still import this subpath and use the +native path or pure path/limit helpers. Some package managers and CI installs skip optional dependencies (`--no-optional`, `--omit=optional`, or equivalent). If an archive helper throws @@ -26,11 +25,16 @@ await extractArchive({ kind: "zip", // optional; resolveArchiveKind() can infer timeoutMs: 15_000, // hard ceiling for the whole extraction stripComponents: 0, // tar-style strip-leading-dirs + entryModes: "clamp", // default; use "preserve" for archive rwx bits + entryFilter: ({ path, kind, size }) => "extract", + onFiltered: "reject-archive", // default; opt into "skip-entry" explicitly limits: { maxArchiveBytes: 256 * 1024 * 1024, maxEntries: 50_000, maxExtractedBytes: 512 * 1024 * 1024, maxEntryBytes: 256 * 1024 * 1024, + maxMetaEntryBytes: 1024 * 1024, + maxEntryPathComponents: 256, }, }); ``` @@ -42,14 +46,67 @@ type ExtractArchiveParams = { archivePath: string; // absolute path to the archive destDir: string; // absolute destination directory; must already exist timeoutMs: number; // wall-clock cap; throws on overrun - kind?: ArchiveKind; // "zip" | "tar"; inferred from filename when omitted + kind?: ArchiveKind; // "zip" | "tar" | "tar-zstd" | "tar-bzip2" stripComponents?: number; // strip N leading dirs from entry paths tarGzip?: boolean; // when archive is .tar.gz/.tgz limits?: ArchiveExtractLimits; logger?: ArchiveLogger; // { info?, warn? } + entryModes?: "clamp" | "preserve"; + entryFilter?: (entry: { path: string; kind: ArchiveEntryKind; size: number }) => + "extract" | "skip"; + onFiltered?: "reject-archive" | "skip-entry"; }; ``` +`entryModes` defaults to `"clamp"`: directories become `0o755`; files become +`0o644`, or `0o755` when the archived owner-execute bit is set. `"preserve"` +keeps archived read/write/execute bits. Both policies strip setuid, setgid, and +sticky bits, and neither applies archived ownership. TAR extraction disables +`tar`'s ownership and mode restoration and applies the selected modes in the +private staging tree; ZIP applies the same policy to `unixPermissions`. + +Native extraction is deliberately split into two phases. Rust first reports an +entry manifest without creating paths. TypeScript validates paths, applies +`stripComponents`, filters, limits, and mode policy, then passes an explicit +accepted-entry plan back to Rust. Rust only performs decompression and the +fd-relative `mkdirBeneath`/exclusive-open writes. This keeps policy identical +between native and JavaScript paths rather than reimplementing it in Rust. + +An `entryFilter` sees the validated archive path, entry kind, and declared +size. Returning `"skip"` rejects the whole archive unless `onFiltered` is +explicitly `"skip-entry"`. Path traversal and archive-wide entry-count checks +still apply to skipped entries. + +For example, a fleet restore can omit regenerated cache entries while rejecting +any other policy mismatch by default: + +```ts +await extractArchive({ + archivePath: snapshotPath, + destDir: restoreRoot, + timeoutMs: 30_000, + entryFilter: ({ path: entryPath, kind }) => + kind === "directory" && entryPath === "state/cache" + ? "skip" + : entryPath.startsWith("state/cache/") + ? "skip" + : "extract", + onFiltered: "skip-entry", + limits: { maxEntries: 50_000, maxEntryPathComponents: 64 }, +}); +``` + +If skipping was not explicitly part of the restore contract, omit +`onFiltered`; the first `"skip"` then rejects the complete archive with +`ArchiveSecurityError("entry-filtered")`. + +Policy rejection is prompt on both implementations. The JavaScript TAR path +owns the file stream and aborts node-tar through a pipeline on filter, path, +link, limit, validation, or timeout failure, which destroys both ends instead +of leaving a paused parser to drain indefinitely. The native path finishes its +bounded manifest read before TypeScript policy evaluation, so a rejected plan +never starts the extraction worker. + If `kind` is omitted, the helper calls `resolveArchiveKind(archivePath)` and throws if the extension is not recognized. Pass `kind` explicitly when the archive name doesn't carry the type (e.g. content-addressed names). ### Limits @@ -60,22 +117,31 @@ type ArchiveExtractLimits = { maxEntries?: number; // refuse before extracting if entry count > this maxExtractedBytes?: number; // refuse mid-stream if total extracted bytes > this maxEntryBytes?: number; // refuse a single entry larger than this + maxMetaEntryBytes?: number; // refuse one PAX/GNU metadata body above this + maxEntryPathComponents?: number; // bound output path depth after stripComponents }; ``` -Defaults exist for each (`DEFAULT_MAX_ARCHIVE_BYTES_ZIP`, `DEFAULT_MAX_ENTRIES`, `DEFAULT_MAX_EXTRACTED_BYTES`, `DEFAULT_MAX_ENTRY_BYTES`). They are conservative — pass explicit values when you know your domain's actual ceiling. +Defaults exist for each (`DEFAULT_MAX_ARCHIVE_BYTES_ZIP`, `DEFAULT_MAX_ENTRIES`, `DEFAULT_MAX_EXTRACTED_BYTES`, `DEFAULT_MAX_ENTRY_BYTES`, `DEFAULT_MAX_META_ENTRY_BYTES`, `DEFAULT_MAX_ENTRY_PATH_COMPONENTS`). The path-component default is 256. It is evaluated after `stripComponents` and before TypeScript accepts an entry for either JavaScript or native extraction, so rejected entries cannot cause implicit parent-directory creation. The 1 MiB metadata default matches node-tar's `maxMetaEntrySize`; fs-safe passes the same resolved value to node-tar and the native TAR meter. A limit violation throws `ArchiveLimitError`. The error's code is one of: ```ts ARCHIVE_LIMIT_ERROR_CODE.ARCHIVE_SIZE_EXCEEDS_LIMIT ARCHIVE_LIMIT_ERROR_CODE.ENTRY_COUNT_EXCEEDS_LIMIT -ARCHIVE_LIMIT_ERROR_CODE.EXTRACTED_BYTES_EXCEEDS_LIMIT -ARCHIVE_LIMIT_ERROR_CODE.ENTRY_BYTES_EXCEEDS_LIMIT +ARCHIVE_LIMIT_ERROR_CODE.EXTRACTED_SIZE_EXCEEDS_LIMIT +ARCHIVE_LIMIT_ERROR_CODE.ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT +ARCHIVE_LIMIT_ERROR_CODE.META_ENTRY_SIZE_EXCEEDS_LIMIT +ARCHIVE_LIMIT_ERROR_CODE.ENTRY_PATH_COMPONENTS_EXCEEDS_LIMIT ``` Catch and branch on the code to surface a meaningful response to the caller. +Entry policy failures throw `ArchiveSecurityError`. Its entry-related codes +are `"entry-path"`, `"entry-link"`, and `"entry-filtered"`; destination-race +codes remain `"destination-not-directory"`, `"destination-symlink"`, and +`"destination-symlink-traversal"`. + ## What it defends against - **Path traversal:** entries with `..`, absolute paths, or Windows drive prefixes are rejected (`ArchiveSecurityError`). @@ -83,6 +149,16 @@ Catch and branch on the code to surface a meaningful response to the caller. - **TOCTOU during merge:** extraction first writes to a private temp dir, then merges into `destDir` using the same boundary checks as `root().write()`. A symlink swap in the destination tree mid-merge is caught. - **Zip bombs:** `maxExtractedBytes` and `maxEntryBytes` apply to *post-decompression* bytes, so highly-compressed payloads hit the cap before they exhaust disk. - **Slow-loris archives:** `timeoutMs` is a hard wall-clock budget. Extraction is aborted on overrun. +- **Metadata bombs:** a fixed-header pass-through reader rejects oversized PAX, GNU long-name, and GNU long-link bodies before either TAR implementation buffers them. It understands octal and base-256 size fields without interpreting metadata content. + +PAX headers can override the next entry's size from inside their content. The +fixed-header meter deliberately never interprets that content, so PAX and GNU +sparse entries are rejected with +`ArchiveFormatError("archive-header-invalid")` rather than guessing. GNU sparse +extension blocks are still metered in 512-byte units before rejection, ensuring +malformed or excessive chains cannot bypass the metadata ceiling. GNU long-name +and long-link entries remain supported because their fixed header size fully +determines their layout. ## `resolveArchiveKind` @@ -91,15 +167,56 @@ import { resolveArchiveKind, type ArchiveKind } from "@openclaw/fs-safe/archive" const kind = resolveArchiveKind("upload.zip"); // "zip" const tar = resolveArchiveKind("upload.tar.gz"); // "tar" +const zstd = resolveArchiveKind("upload.tar.zst"); // "tar-zstd" when native is available const unknown = resolveArchiveKind("upload.bin"); // undefined ``` Recognizes: - `*.zip` → `"zip"` -- `*.tar`, `*.tar.gz`, `*.tgz`, `*.tar.bz2`, `*.tbz`, `*.tbz2` → `"tar"` +- `*.tar`, `*.tar.gz`, `*.tgz` → `"tar"` +- `*.tar.zst`, `*.tar.zstd`, `*.tzst` → `"tar-zstd"` (native only) +- `*.tar.bz2`, `*.tbz2`, `*.tbz` → `"tar-bzip2"` (native only) -Returns `undefined` for unknown extensions; check the result before calling `extractArchive` if the filename is caller-controlled. +Returns `undefined` for unknown extensions; check the result before calling +`extractArchive` if the filename is caller-controlled. A recognized zstd or +bzip2 TAR extension with no native binding throws the typed +`FsSafeError("helper-unavailable")` with installation guidance. This includes +`mode: "off"`; those two formats have no JavaScript fallback. + +For a service whose input contract requires zstd, configure native mode before +the first archive call so a packaging mistake fails at the boundary: + +```ts +import { configureFsSafeNative } from "@openclaw/fs-safe/config"; +import { extractArchive } from "@openclaw/fs-safe/archive"; + +configureFsSafeNative({ mode: "require" }); +await extractArchive({ + archivePath: "/srv/restore/snapshot.tar.zst", + destDir: "/srv/restore/staging", + kind: "tar-zstd", + timeoutMs: 60_000, +}); +``` + +## `readArchiveEntry` + +`readArchiveEntry(archivePath, entryPath, { maxBytes, kind? })` reads one +regular-file entry into a bounded `Buffer` without extracting a tree. It pins +and privately stages the archive input, rejects link and directory entries, +and throws `ArchiveLimitError` if decompressed bytes exceed `maxBytes`. ZIP +inputs retain the archive subpath's 256 MiB compressed-input ceiling. +With a native binding it uses the same Rust decoders as extraction, including +zstd and bzip2 TAR. Without native it retains the JS ZIP/TAR/gzip implementation. + +```ts +const rawManifest = await readArchiveEntry(uploadPath, "package/manifest.json", { + maxBytes: 64 * 1024, +}); +const manifest = JSON.parse(rawManifest.toString("utf8")) as PluginManifest; +validatePluginManifest(manifest); +``` ## Lower-level building blocks @@ -196,4 +313,5 @@ await withTempWorkspace({ rootDir: "/srv/site/tmp", prefix: "extract-" }, async - [Atomic writes](atomic.md) — `replaceDirectoryAtomic` for staged directory replacement. - [Temp workspaces](temp.md) — extract into a private workspace and commit as one step. - [Errors](errors.md) — `FsSafeError` codes the underlying writes can raise. +- [Migrating to 0.5](migrating-to-0.5.md) — clamp-default and native-format upgrade checklist. - [`extractArchive` source](https://github.com/openclaw/fs-safe/blob/main/src/archive.ts). diff --git a/docs/atomic.md b/docs/atomic.md index deb56f0..00ec59d 100644 --- a/docs/atomic.md +++ b/docs/atomic.md @@ -148,6 +148,13 @@ await movePathWithCopyFallback({ ``` Use it when source and destination might live on different filesystems (containers, tmpfs, separate volumes). +`sourceHardlinks: "reject"` performs a recursive preflight capped at 50,000 +entries before any mutation. Because link count and rename cannot be one atomic +portable operation, this mode always commits a fresh inode/tree through the +staged-copy route, even on one filesystem. Each regular file is checked again +after open and after copying, so a post-scan hardlink cannot become the +published target. A hardlink fails with `FsSafeError("hardlink")`; exceeding +the preflight cap fails with `FsSafeError("too-large")`. If another writer changes source entries during the fallback, the staged copy throws `ESTALE` before commit when possible. If the destination has already been committed, cleanup still preserves the changed source entries and throws @@ -160,7 +167,7 @@ been committed, cleanup still preserves the changed source entries and throws | Take relative paths, bound to a `rootDir`. | Take absolute paths, no boundary. | | Throw `FsSafeError` with `code`. | Throw `FsSafeError` *or* the underlying `NodeJS.ErrnoException`, depending on failure point. | | Atomicity, mode, hooks, fsync are sane defaults. | Caller controls all of the above. | -| `mkdir`, identity check, hardlink reject built in. | No identity check, no hardlink reject — pair with [path helpers](path.md) if you need them. | +| `mkdir`, identity check, hardlink reject built in. | No root boundary; `movePathWithCopyFallback` has explicit `sourceHardlinks` policy, while other helpers expose their own narrower checks. | Use `Root` when the path is caller-controlled. Use `atomic` when the path is fully under your control and you want explicit knobs. diff --git a/docs/config.md b/docs/config.md index 4b41894..8ac2bcc 100644 --- a/docs/config.md +++ b/docs/config.md @@ -5,47 +5,46 @@ description: "Process-global defaults for optional fs-safe helpers." # `@openclaw/fs-safe/config` -Process-global configuration knobs for optional fs-safe helpers. The Python helper policy is described in the [Python helper policy](python-helper.md); this page is the API reference. +Process-global configuration knobs for optional fs-safe helpers. The native helper policy is described in the [native helper policy](native-helper.md); this page is the API reference. ```ts import { - configureFsSafePython, + configureFsSafeNative, configureFsSafeLocks, - getFsSafePythonConfig, + getFsSafeNativeConfig, getFsSafeLockConfig, type FsSafeLockConfig, - type FsSafePythonConfig, - type FsSafePythonMode, + type FsSafeNativeConfig, + type FsSafeNativeMode, } from "@openclaw/fs-safe/config"; ``` These functions are also re-exported from the main entry point. Prefer the subpath when you only need helper configuration and want the smallest import surface. -## `configureFsSafePython(config)` +## `configureFsSafeNative(config)` ```ts -function configureFsSafePython(config: Partial): void; +function configureFsSafeNative(config: Partial): void; -type FsSafePythonConfig = { - mode: FsSafePythonMode; - pythonPath?: string; +type FsSafeNativeConfig = { + mode: FsSafeNativeMode; }; -type FsSafePythonMode = "auto" | "off" | "require"; +type FsSafeNativeMode = "auto" | "off" | "require"; ``` -Set the process-global policy. Calls merge into the existing override config, so passing `{ pythonPath: "/usr/bin/python3" }` keeps any previously set `mode`. Configure once at startup, before the first `root()` call — switching modes mid-process is supported but the helper may already be running. +Set the process-global loading policy. Configure once at startup, before the first filesystem operation. The binding is loaded lazily and the result is cached. | Mode | Behavior | |---|---| -| `auto` | Default. Use the helper when it starts; fall back to Node-only behavior if Python is missing or fails to start. | -| `off` | Never spawn the helper. Read/write/move use Node fallbacks plus pre/post identity checks. | -| `require` | Fail closed if the helper cannot start. Operations that need the helper raise `FsSafeError("helper-unavailable")`. | +| `auto` | Default. Prefer the platform binding and use guarded JavaScript when it is unavailable. | +| `off` | Do not load the binding; use guarded JavaScript deterministically. | +| `require` | Operations that need the binding raise `FsSafeError("helper-unavailable")` when it cannot load. | -## `getFsSafePythonConfig()` +## `getFsSafeNativeConfig()` ```ts -function getFsSafePythonConfig(): FsSafePythonConfig; +function getFsSafeNativeConfig(): FsSafeNativeConfig; ``` Return the effective configuration: programmatic overrides win, then env vars, then the package default (`auto`). @@ -67,6 +66,22 @@ Set process-wide defaults for sidecar lock options. This does **not** turn locki `staleRecovery` defaults to `"fail-closed"`. The opt-in `"remove-if-unchanged"` mode requires caller approval and serializes the final snapshot check and unlink with an exclusive `.reclaim` guard. A reclaim guard left by a killed reclaimer fails closed and requires externally coordinated cleanup. +For a daemon that should wait briefly for normal contention but never delete a +stale owner without per-lock approval: + +```ts +configureFsSafeLocks({ + staleRecovery: "fail-closed", + staleMs: 2 * 60_000, + timeoutMs: 15_000, + retry: { retries: 30, minTimeout: 50, maxTimeout: 1_000, randomize: true }, +}); +``` + +Individual lock calls can override any default. Switching the global stale +recovery mode does not provide the application-owned liveness proof required +by `shouldRemoveStaleLock`. + ## `getFsSafeLockConfig()` ```ts @@ -80,15 +95,30 @@ Return the current sidecar lock defaults. The same policy can be set without code: ```bash -FS_SAFE_PYTHON_MODE=auto # auto | off | require | true | false | on | off | 1 | 0 | never | required -FS_SAFE_PYTHON=/usr/bin/python3 +FS_SAFE_NATIVE_MODE=auto # auto | off | require | true | false | on | 1 | 0 | never | required ``` -OpenClaw compatibility aliases are accepted: `OPENCLAW_FS_SAFE_PYTHON_MODE`, `OPENCLAW_FS_SAFE_PYTHON`, `OPENCLAW_PINNED_PYTHON`, and `OPENCLAW_PINNED_WRITE_PYTHON`. Programmatic overrides via `configureFsSafePython` always win. +`OPENCLAW_FS_SAFE_NATIVE_MODE` is accepted as an alias. Programmatic overrides via `configureFsSafeNative` always win. + +### Python-helper migration bridge + +Version 0.5 detects the former `FS_SAFE_PYTHON_MODE`, `FS_SAFE_PYTHON`, +`OPENCLAW_FS_SAFE_PYTHON_MODE`, `OPENCLAW_FS_SAFE_PYTHON`, +`OPENCLAW_PINNED_PYTHON`, and `OPENCLAW_PINNED_WRITE_PYTHON` names. It emits one +`FS_SAFE_PYTHON_DEPRECATED` warning and maps `auto`, `off`, or `require` to the +same native mode; interpreter paths are ignored. The deprecated +`configureFsSafePython()` export behaves the same way. + +Replace these inputs with `configureFsSafeNative()` or +`FS_SAFE_NATIVE_MODE` during the 0.5 upgrade. The bridge exists only so shipped +0.4 configuration fails loudly and maps predictably; it is not a supported +Python execution path. Follow the [0.5 migration checklist](migrating-to-0.5.md) +for the full upgrade. ## Related pages -- [Python helper policy](python-helper.md) — when to pick `auto`, `off`, or `require`, and what each mode protects. +- [Native helper policy](native-helper.md) — when to pick `auto`, `off`, or `require`, and what each mode protects. - [File lock](sidecar-lock.md) — the per-resource lock API that consumes lock defaults. - [Root API](root.md) — the API whose POSIX hardening the helper backs. - [Errors](errors.md) — `helper-unavailable` and `helper-failed`. +- [Migrating to 0.5](migrating-to-0.5.md) — ordered consumer upgrade checklist. diff --git a/docs/durability.md b/docs/durability.md index 746c7ea..94f7bea 100644 --- a/docs/durability.md +++ b/docs/durability.md @@ -81,9 +81,187 @@ accepted. the caller before a separate permission or policy check. A missing or replaced target fails with `FsSafeError("path-mismatch")`. +## Exclusive file publication + +`publishFileExclusive()` materializes one file without clobbering an existing +target. It pins the source with `O_NOFOLLOW`, optionally verifies +`expectedSourceIdentity`, tries a hardlink first, then synchronizes the target +parent directory. + +For example, a backup archive is complete before publication. If directory +sync fails, keeping that complete file is more useful than conditionally +deleting it by pathname: + +```ts +import { FsSafeError } from "@openclaw/fs-safe/errors"; +import { publishFileExclusive } from "@openclaw/fs-safe/durability"; + +try { + const result = await publishFileExclusive({ + sourcePath: stagedArchive, + targetPath: finalArchive, + strategy: "link-or-copy", + onSyncFailure: "preserve", + parentReceipt: backupDirectory, + }); + recordDurableBackup(result.identity, result.directorySync); +} catch (error) { + if ( + error instanceof FsSafeError && + error.details?.phase === "directory-sync" && + error.details.cleanup === "preserved" + ) { + recordCompleteButPossiblyNonDurableBackup(finalArchive, error.details); + } else { + throw error; + } +} +``` + +### Strategies + +| Strategy | Behavior | Native requirement | +|---|---|---| +| `link-required` | Create a same-filesystem hardlink or propagate the failure. | No; guarded JS `link` fallback remains. | +| `link-or-copy` | Try hardlink, then clone, Linux `copy_file_range`, then the JS byte loop for classified unsupported errors. | No; acceleration is optional. | +| `rename-noreplace` | Atomically move the source without replacing an existing target. Success consumes `sourcePath`. | Yes. | + +`"link-required"` propagates an unsupported hardlink failure. +`"link-or-copy"` falls back only for `EPERM`, `EXDEV`, `ENOTSUP`, +`EOPNOTSUPP`, or `ENOSYS`; `isHardlinkFallbackError()` exposes that exact +classifier. The fallback copies from the pinned source into a `wx` target, +fsyncs it, and fences source and target identity and content before reporting +success. `parentReceipt`, when supplied, must name the target's direct parent. + +With a native binding, the copy fallback first attempts a copy-on-write clone +(`fclonefileat` on macOS, `FICLONE` on Linux), then Linux +`copy_file_range`, and finally the existing JavaScript byte loop. Every route +creates the target exclusively, normalizes its mode to `0o600`, and goes +through the same post-copy identity and SHA-256 fencing. Hashing uses an async +native task when available, so large verification reads do not occupy the +JavaScript event loop. + +On a clone-capable filesystem, publication of a large file becomes mostly a +metadata operation: data blocks are shared copy-on-write until either file is +modified. Clone support is filesystem- and mount-dependent, so callers must +not infer durability or physical independence from timing; an unsupported +clone or `copy_file_range` transparently continues down the fallback chain. + +## Streaming SHA-256 + +`sha256File()` hashes either a pathname string or an already-open Node +`FileHandle`. A backup verifier can pin the file itself, compare its size, and +keep ownership of the handle: + +```ts +import { open } from "node:fs/promises"; +import { sha256File } from "@openclaw/fs-safe/durability"; + +const snapshot = await open(stagedArchive, "r"); +try { + const before = await snapshot.stat(); + const hash = await sha256File(snapshot); + if (hash.bytes !== before.size || hash.digest !== manifest.sha256) { + throw new Error("staged backup does not match its manifest"); + } +} finally { + await snapshot.close(); +} +``` + +The result is `{ bytes, digest }`, where `digest` is lowercase hexadecimal. +The handle overload never closes the caller's descriptor and uses positioned +reads, so it does not alter the descriptor's current offset. The path overload +rejects symbolic links and non-regular files, verifies the opened descriptor +still names the requested path, and closes its own handle. POSIX opens are +nonblocking, so a raced FIFO or device is rejected after descriptor inspection +rather than waiting for a writer. + +When the optional binding is active, hashing runs as an async native task and +does not occupy the JavaScript event loop with digest updates. With native mode +`off`, or in `auto` when no binding loads, the fallback performs asynchronous +positioned reads in 64 KiB chunks but updates Node's `Hash` on the JavaScript +thread. Both paths stream constant-size buffers rather than loading the file +into memory. Native mode `require` keeps its usual fail-closed loader semantics. + +If publication fails after this call created the target, it throws an +`FsSafeError` with a `details` receipt: + +```ts +type PublishFileExclusiveFailureDetails = { + phase: + | "hardlink-create" | "hardlink-verify" + | "copy-create" | "copy-verify" + | "rename-create" | "rename-verify" + | "directory-sync"; + targetCreated: boolean; + targetIdentity?: { dev: number | bigint; ino: number | bigint }; + cleanup: "removed" | "preserved" | "unknown"; + directorySync?: { status: "failed"; code?: string }; +}; +``` + +`"removed"` means the path still matched the identity created by this call and +was unlinked (or was already absent). `"preserved"` means it was deliberately +retained—for example after a successful no-replace rename—or the pathname had +been replaced and therefore was not safe to remove. `"unknown"` means cleanup +could not verify or remove the created identity. Callers that run a second +application-level guard, such as SQLite snapshot validation, should branch on +this receipt instead of inferring ownership from path existence. The original +failure remains available as `cause`. Failures before target creation retain +their existing error shape and do not claim a cleanup result. + +### Directory-sync failure policy + +`onSyncFailure` applies only after target creation and content/identity fencing +have succeeded but synchronizing the containing directory throws: + +```ts +type PublishFileExclusiveSyncFailurePolicy = "rollback" | "preserve"; +``` + +A returned `{ status: "unsupported", code? }` is an explicit successful +publication outcome, not a thrown sync failure, so this option does not rewrite +or clean up that target. + +- `rollback` is the default. fs-safe removes the target only if its current + identity still matches the file created by this call. A replacement is never + removed. The error reports `cleanup: "removed"`, `"preserved"`, or + `"unknown"` and `directorySync: { status: "failed", code? }`. +- `preserve` never attempts that unlink. The error reports + `targetCreated: true`, `cleanup: "preserved"`, the created identity, and the + failed directory-sync outcome. The file is complete and fenced, but its + directory entry is not proven crash-durable. + +Choose `rollback` when the pathname must mean “durably committed” and a failed +commit should disappear from the live process view. Choose `preserve` when the +payload itself remains valuable—backup archives are the common case—and the +caller can record, retry, or independently validate durability. Neither choice +can make a failed directory sync succeed: rollback deletion is also not proven +durable, and a preserved name may disappear after a crash. Always use the +typed receipt rather than inferring ownership from `exists()`. + +`rename-noreplace` always preserves its target after a successful rename, +because removing it would discard the source's only remaining name; its typed +failure receipt makes that explicit regardless of `onSyncFailure`. + +`"rename-noreplace"` requires the native helper and atomically moves the +source to the target without replacement. A collision is reported as +`EEXIST`, both files remain unchanged, and a successful call returns +`method: "rename-noreplace"` after synchronizing the source and target parent +directories. Unlike the link/copy strategies, success consumes `sourcePath`. + ## Scope -These primitives establish path identity and filesystem synchronization. They +These primitives establish path identity and filesystem synchronization. One +`publishFileExclusive()` call is one no-clobber file materialization, not a +retention policy, multi-file transaction, or application commit protocol. They do not decide application commit protocols, marker formats, permission policy, or whether an unsupported platform is acceptable. Keep those decisions at the owning product boundary. + +## See also + +- [Migrating to 0.5](migrating-to-0.5.md) — choosing a publication policy during upgrade. +- [Native architecture](native.md) — clone/copy/hash mechanisms and fallback guarantees. +- [Errors](errors.md) — typed operational failure handling. diff --git a/docs/errors.md b/docs/errors.md index e19b6c0..de63640 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -13,13 +13,24 @@ class FsSafeError extends Error { readonly name: "FsSafeError"; readonly code: FsSafeErrorCode; readonly category: "policy" | "operational"; + readonly details?: Readonly>; - constructor(code: FsSafeErrorCode, message: string, options?: { cause?: unknown }); + constructor( + code: FsSafeErrorCode, + message: string, + options?: { cause?: unknown; details?: Readonly> }, + ); } ``` `cause` is available through the standard `Error` `cause` property when the failure was triggered by a `NodeJS.ErrnoException` (e.g. a wrapped `EACCES`). Inspect it for the original `code` / `errno` / `syscall` if you need finer-grained reporting. +`details` is an operation-specific receipt, not an alternate error code. For +example, `publishFileExclusive()` uses it to report the failing phase, created +target identity, cleanup decision, and failed directory-sync outcome. Narrow +by `code` and a documented details field before consuming it; do not assume all +`FsSafeError` instances carry the same keys. + `category` separates caller-policy failures from operational failures: - `"policy"` — unsafe input or target state, such as `outside-workspace`, `symlink`, `hardlink`, or `too-large`. @@ -46,6 +57,7 @@ type FsSafeErrorCode = | "path-alias" | "path-mismatch" | "permission-unverified" + | "secret-exists" | "symlink" | "timeout" | "too-large" @@ -60,8 +72,8 @@ type FsSafeErrorCode = | `denied-path` | A root mutation matched `denyMutations.paths` or `denyMutations.prefixes`. | Caller configured application-sensitive paths that must not be written, removed, moved, or created. | | `device-path` | A read/open target is a known unsafe device or process-fd path. | `/dev/zero`, `/dev/random`, `/dev/stdin`, `/dev/fd/*`, `/proc/*/fd/*`, or a Windows reserved device name. | | `hardlink` | Read or copy with `hardlinks: "reject"` saw `nlink > 1`. | File is hardlinked — possibly an alias of an out-of-tree inode. | -| `helper-failed` | Internal POSIX helper failed after startup. | Inspect `cause`; retrying may be unsafe if the operation may have partially completed. | -| `helper-unavailable` | Persistent Python helper was disabled or could not be spawned. | `FS_SAFE_PYTHON_MODE=off`, Python missing in PATH, restricted sandbox. `auto` falls back where possible; `require` fails closed. | +| `helper-failed` | A native mechanism or multi-step operational helper failed. | Inspect `cause` and any operation-specific `details`; retrying may be unsafe if the operation partially completed. | +| `helper-unavailable` | Required native binding could not be loaded. | Missing/incompatible optional platform package or `FS_SAFE_NATIVE_MODE=require`. `auto` falls back where possible; `require` fails closed. | | `insecure-permissions` | A secure file or path permission check found a mode/ACL that allows broader access than requested. | File or directory is group/world writable/readable; Windows ACL grants broad read. | | `invalid-path` | Input was empty, contained NUL, was an unparseable URL, or otherwise unusable. | Caller didn't validate input; input was a network path on Windows. | | `not-empty` | `remove()` on a non-empty directory. | Use `replaceDirectoryAtomic` or remove children first. | @@ -73,9 +85,10 @@ type FsSafeErrorCode = | `path-alias` | A path alias check failed (e.g. canonical-real-path moved out of the root). | Symlink resolution lands outside the root. | | `path-mismatch` | Post-open identity check failed: the opened fd does not match the resolved path. | TOCTOU — something else swapped the path between resolve and open. | | `permission-unverified` | A secure file check could not verify required permissions. | Windows ACL inspection failed; POSIX ownership/mode was unavailable. | +| `secret-exists` | `createSecretFileAtomic()` found an existing final path. | First-writer-wins secret creation lost a race or the credential was already initialized. | | `symlink` | Path component is a symlink, policy is `reject`. | Caller followed a symlink they shouldn't have, or `symlinks: "reject"` is set. | | `timeout` | An operation with a wall-clock budget overran. | Secure file read or timed operation exceeded `timeoutMs`. | -| `too-large` | Read exceeded `maxBytes`. | Caller gave a too-permissive file or didn't size-cap correctly. | +| `too-large` | A read or bounded walk exceeded its configured budget. | Caller gave a too-permissive file or traversal limit. | | `unsupported-platform` | The requested operation is not supported on the current platform. | E.g. POSIX-only helper invoked on Windows. | ## Branching @@ -139,8 +152,8 @@ A common pattern is to wrap your domain code in a single try/catch that maps bot A handful of helpers throw their own typed errors instead of `FsSafeError`: - `JsonFileReadError` — thrown by [`readJson`](json.md). Carries `cause` so you can distinguish missing (`ENOENT`) from invalid (`SyntaxError`). -- `ArchiveLimitError` — thrown by [`extractArchive`](archive.md) when an archive size, entry count, or extracted-byte budget is exceeded. The `code` field uses `ARCHIVE_LIMIT_ERROR_CODE` constants (e.g. `"ARCHIVE_SIZE_EXCEEDS_LIMIT"`). -- `ArchiveSecurityError` — thrown by extraction when an entry path violates safety rules (traversal, drive prefix, blocked link type). The `code` field uses `ArchiveSecurityErrorCode` values. +- `ArchiveLimitError` — thrown by [`extractArchive`](archive.md) when an archive size, entry count, path depth, metadata, or extracted-byte budget is exceeded. The `code` field uses the string values exposed by `ARCHIVE_LIMIT_ERROR_CODE` (for example `archive-entry-path-components-exceeds-limit`). +- `ArchiveSecurityError` — thrown by extraction when entry policy or destination safety fails. Entry codes are `entry-path`, `entry-link`, and `entry-filtered`; destination codes cover non-directory, symlink, and symlink-traversal failures. These are exported from their respective subpaths. diff --git a/docs/index.md b/docs/index.md index 43df9d5..bf2aab9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -38,7 +38,8 @@ await fs.remove("notes/archive/today.txt"); ## Pick your path - **First time?** [Install](install.md), then walk through the [Quickstart](quickstart.md). Five minutes from `pnpm add` to a working root. -- **Designing a workspace feature.** Read the [Security model](security-model.md) before you trust the boundary, the [Python helper policy](python-helper.md) before you pick deployment defaults, and the [Errors](errors.md) reference so you know what to catch. +- **Upgrading from 0.4?** Follow [Migrating to 0.5](migrating-to-0.5.md) in order, including the archive clamp-default audit. +- **Designing a workspace feature.** Read the [Security model](security-model.md) before you trust the boundary, the [native helper policy](native-helper.md) before you pick deployment defaults, and the [Errors](errors.md) reference so you know what to catch. - **Replacing ad-hoc atomic writes.** Jump to [Atomic writes](atomic.md) or, for keyed JSON state, [JSON files](json.md). - **Extracting an upload.** Start at [Archive extraction](archive.md) — handles ZIP and TAR with traversal, link, count, and byte limits. - **Running an agent in a sandbox.** [Private temp workspaces](temp.md) plus [secret files](secret-file.md) cover the common scratch-and-credentials shape. @@ -48,11 +49,12 @@ await fs.remove("notes/archive/today.txt"); | Surface | Use it for | |---|---| -| [`root()`](root.md) | One boundary for read/write/move/remove inside a trusted directory. | -| [`@openclaw/fs-safe/config`](config.md) | Process-global Python helper and lock-option defaults. | -| [Python helper policy](python-helper.md) | Choose `auto`, `off`, or `require` for POSIX fd-relative hardening. | +| [`root()`](root.md) | One boundary for read/write/move/remove and bounded recursive walking inside a trusted directory. | +| [`@openclaw/fs-safe/config`](config.md) | Process-global native helper and lock-option defaults. | +| [Native helper policy](native-helper.md) | Choose `auto`, `off`, or `require` for optional native primitives. | +| [Native architecture](native.md) | Understand the thin syscall layer, beneath model, platform mechanisms, and fallback boundary. | | [`replaceFileAtomic`](atomic.md) | Sibling-temp + rename, fsync hooks, mode preservation, copy fallback. | -| [Directory durability](durability.md) | Pinned directory identities, explicit sync outcomes, and durable nested-directory creation. | +| [`@openclaw/fs-safe/durability`](durability.md) | Pinned directory identities, durable creation, exclusive publication, streaming SHA-256, provenance receipts, and sync-failure policy. | | [`writeExternalFileWithinRoot`](output.md) | Stage external-library file output in private temp storage, then finalize under a root. | | [`writeJson` / `readJson*`](json.md) | JSON state files with strict and lenient read variants. | | [`@openclaw/fs-safe/store`](store.md) | Overview of `fileStore`, `fileStoreSync`, and `jsonStore`. | @@ -61,15 +63,16 @@ await fs.remove("notes/archive/today.txt"); | [Private file-store mode](private-file-store.md) | `fileStore({ private: true })` for private JSON/text state at 0600 under 0700 dirs. | | [`tempWorkspace`](temp.md) | 0700 scratch dir with auto-cleanup. | | [`readSecureFile`](secure-file.md) | Absolute file reads with fd pinning, permissions, owner, size, and timeout checks. | -| [`walkDirectory`](walk.md) | Budget-bounded recursive directory scan with symlink policy and filters. | -| [`extractArchive`](archive.md) | ZIP/TAR extraction with size, count, link, and traversal limits. | +| [`walkDirectory` / `Root.walk`](walk.md) | Standalone inventories plus root-bounded pruning, budgets, and partial-error reporting. | +| [`extractArchive`](archive.md) | Policy-driven ZIP/TAR extraction with clamp/filter, metadata/path-depth, link, count, and byte limits. | | [Secret files](secret-file.md) | Mode-0600 credentials with size and TOCTOU defense. | -| [Permissions](permissions.md) | POSIX mode and Windows ACL inspection/remediation helpers. | +| [Permissions](permissions.md) | POSIX mode helpers plus Windows ACL inspection, raw owner/ACE facts, remediation, and private-directory creation. | | [`acquireFileLock`](sidecar-lock.md) | Cross-process file lock with retry and fail-closed stale-lock handling. | | [`FsSafeError`](errors.md) | Closed code union (with `policy` / `operational` category) you can branch on. | | [`pathScope()`](path-scope.md) | Lower-level absolute-path boundary helper; lives behind `@openclaw/fs-safe/advanced`. | | [`@openclaw/fs-safe/advanced`](advanced.md) | Directory of lower-level composition helpers (path scopes, regular-file I/O, install paths, sibling-temp writes, …). | | [`@openclaw/fs-safe/test-hooks`](test-hooks.md) | Test-only injection hooks for reproducing open/lstat races. | +| [Migrating to 0.5](migrating-to-0.5.md) | End-to-end checklist for Python removal and 0.4 API behavior changes. | ## Status diff --git a/docs/install.md b/docs/install.md index 1942f33..809b730 100644 --- a/docs/install.md +++ b/docs/install.md @@ -62,20 +62,22 @@ Use the main entry for the common surface, or the focused subpaths when you want | Subpath | Contents | |---|---| -| `@openclaw/fs-safe` | Small common surface: `root`, root types, and errors. | -| `@openclaw/fs-safe/root` | `root()`, `Root`, `RootDefaults`, related types. | -| `@openclaw/fs-safe/config` | Process-global Python helper configuration. | +| `@openclaw/fs-safe` | Common root, config, output, lock, native-mode, and error exports. | +| `@openclaw/fs-safe/root` | `root()`, `Root`, `RootDefaults`, and root-walk types. | +| `@openclaw/fs-safe/config` | Process-global native helper and lock defaults. | | `@openclaw/fs-safe/path` | `isPathInside`, `safeRealpathSync`, `isWithinDir`, error helpers. | +| `@openclaw/fs-safe/output` | Guarded staging/finalization for libraries that require an absolute output path. | | `@openclaw/fs-safe/json` | `tryReadJson`, `readJson`, `readJsonIfExists`, `writeJson`, sync variants. | | `@openclaw/fs-safe/store` | `fileStore()`, `fileStoreSync()`, and `jsonStore()`. | | `@openclaw/fs-safe/secret` | Secret file read/write helpers. | | `@openclaw/fs-safe/atomic` | `replaceFileAtomic`, `writeTextAtomic`, `replaceDirectoryAtomic`, `movePathWithCopyFallback`. | +| `@openclaw/fs-safe/durability` | Pinned directories, strict sync, durable directory creation, exclusive publication, and streaming SHA-256. | | `@openclaw/fs-safe/temp` | `tempWorkspace`, `withTempWorkspace`, sync variants, `resolveSecureTempRoot`. | | `@openclaw/fs-safe/secure-file` | `readSecureFile` for pinned absolute file reads with permissions checks. | | `@openclaw/fs-safe/file-lock` | `acquireFileLock`, `withFileLock`, `createFileLockManager`, and related lock types. | -| `@openclaw/fs-safe/permissions` | POSIX mode and Windows ACL inspection/remediation helpers. | +| `@openclaw/fs-safe/permissions` | POSIX mode helpers, Windows ACL inspection/remediation, raw owner/ACE facts, and private-directory creation. | | `@openclaw/fs-safe/walk` | `walkDirectory`, `walkDirectorySync`, related types. Budget-bounded, not root-bounded. | -| `@openclaw/fs-safe/archive` | `extractArchive`, `resolveArchiveKind`, limits, preflight helpers. | +| `@openclaw/fs-safe/archive` | `extractArchive`, `readArchiveEntry`, kind resolution, policy types, limits, and preflight helpers. | | `@openclaw/fs-safe/advanced` | Lower-level composition helpers: path scopes, root-file open, install paths, local-root readers, temp-file targets, sibling-temp writes, regular-file helpers, `pathExists`, `withTimeout`, and related advanced types. This surface is less stable than the focused public subpaths. | | `@openclaw/fs-safe/errors` | `FsSafeError`, `FsSafeErrorCode`. | | `@openclaw/fs-safe/types` | Shared types: `DirEntry`, `PathStat`, `BasePathOptions`, … | @@ -85,40 +87,37 @@ Use the main entry for the common surface, or the focused subpaths when you want `@openclaw/fs-safe` lists `jszip` and `tar` as optional dependencies for [archive extraction](archive.md). They are loaded lazily and only required when ZIP/TAR helpers run. Installs that omit optional dependencies can still import and use every non-archive subpath; archive calls fail with a clear missing-optional-dependency message. -There are no peer dependencies and no native build step. +There are no peer dependencies. Native helpers are optional prebuilt packages, so consumers do not run a native build during installation. -## Python helper policy +Upgrading an existing consumer? Follow [Migrating to 0.5](migrating-to-0.5.md) +before choosing a native mode or accepting the new archive clamp default. -On POSIX, `root()` uses one persistent Python helper process for the -fd-relative operations Node does not expose cleanly. The default is `auto`: use -the helper when it starts, fall back to Node-only behavior when it is disabled -or unavailable. +## Native helper policy + +The optional native package provides fd-relative open/link/mkdir primitives, +atomic no-replace rename, and file identity checks. The default is `auto`: use +the platform binding when it loads, otherwise keep the guarded JavaScript path. ```ts -import { configureFsSafePython } from "@openclaw/fs-safe/config"; +import { configureFsSafeNative } from "@openclaw/fs-safe/config"; -configureFsSafePython({ mode: "auto" }); // default -configureFsSafePython({ mode: "off" }); // never spawn Python -configureFsSafePython({ mode: "require" }); // fail closed if unavailable +configureFsSafeNative({ mode: "auto" }); // default +configureFsSafeNative({ mode: "off" }); // guarded JavaScript only +configureFsSafeNative({ mode: "require" }); // fail closed if unavailable ``` Environment variables are read at runtime: ```bash -FS_SAFE_PYTHON_MODE=off # auto | off | require -FS_SAFE_PYTHON=/usr/bin/python3 +FS_SAFE_NATIVE_MODE=off # auto | off | require ``` -OpenClaw compatibility aliases are also accepted: -`OPENCLAW_FS_SAFE_PYTHON_MODE`, `OPENCLAW_FS_SAFE_PYTHON`, -`OPENCLAW_PINNED_PYTHON`, and `OPENCLAW_PINNED_WRITE_PYTHON`. +`OPENCLAW_FS_SAFE_NATIVE_MODE` is also accepted. -Disabling Python keeps the public API working, but downgrades POSIX mutation -hardening from fd-relative syscalls to Node path operations guarded by lexical -and canonical checks plus identity verification. Use `require` for -security-sensitive deployments where that downgrade should be a startup/runtime -failure instead of a fallback. The full tradeoff is documented in -[Python helper policy](python-helper.md). +Disabling native loading keeps the public API working through Node path +operations guarded by lexical and canonical checks plus identity verification. +Use `require` when native-backed operations must fail instead of falling back. +The exact boundary is documented in [native helper policy](native-helper.md). ## Verify the install diff --git a/docs/migrating-to-0.5.md b/docs/migrating-to-0.5.md new file mode 100644 index 0000000..6389fc1 --- /dev/null +++ b/docs/migrating-to-0.5.md @@ -0,0 +1,182 @@ +--- +title: Migrating to 0.5 +description: "Ordered checklist for moving a 0.4 consumer from the Python helper to native mode and adopting the 0.5 API contracts." +--- + +# Migrating from 0.4 to 0.5 + +Use this checklist from top to bottom. Version 0.5 replaces the Python worker, +changes the default archive mode policy, and adds explicit contracts for +publication, walking, locks, secrets, and native-only features. Nothing in this +guide requires a Rust toolchain: supported native packages are prebuilt and +optional. + +## 1. Update the package and runtime + +- Run on Node.js 22 or newer. +- Update `@openclaw/fs-safe` and regenerate every lock or shrinkwrap file your + deployment consumes. +- Keep optional dependencies enabled if you want automatic native loading and + JavaScript ZIP/TAR support. An install that omits optional packages can still + import fs-safe, but native-only features and missing JS archive decoders fail + with actionable errors. + +## 2. Replace Python helper configuration + +Change startup configuration before the first filesystem operation: + +```ts +import { configureFsSafeNative } from "@openclaw/fs-safe/config"; + +configureFsSafeNative({ mode: "auto" }); +``` + +| Remove from 0.4 | Use in 0.5 | +|---|---| +| `configureFsSafePython({ mode })` | `configureFsSafeNative({ mode })` | +| `FS_SAFE_PYTHON_MODE` | `FS_SAFE_NATIVE_MODE` | +| `OPENCLAW_FS_SAFE_PYTHON_MODE` | `OPENCLAW_FS_SAFE_NATIVE_MODE` | +| `pythonPath`, `FS_SAFE_PYTHON`, pinned-Python aliases | Nothing; native packages do not use an interpreter | + +The old names warn once and map `auto`, `off`, or `require` so a shipped 0.4 +deployment does not silently change policy. Interpreter paths are ignored and +Python is never spawned. Treat that warning as an upgrade diagnostic, not as a +second supported helper path. + +Choose the production mode deliberately: + +- `auto` keeps guarded JavaScript fallbacks when a binding is unavailable. +- `off` makes fallback testing deterministic. +- `require` fails with `helper-unavailable` instead of weakening an operation + that expected native support. + +See [Native helper policy](native-helper.md) and +[Native architecture](native.md). + +## 3. Audit every archive call + +The 0.5 default is `entryModes: "clamp"`. Directories become `0o755`; files +become `0o644` or `0o755` when owner-execute was archived. Set +`entryModes: "preserve"` explicitly only if your 0.4 consumer intentionally +relied on archived rwx bits. Setuid, setgid, sticky bits, and archived ownership +are never restored. + +```ts +await extractArchive({ + archivePath: uploadPath, + destDir: restoreRoot, + timeoutMs: 30_000, + entryModes: "clamp", + entryFilter: (entry) => + entry.path.startsWith("snapshot/cache/") ? "skip" : "extract", + onFiltered: "skip-entry", + limits: { + maxArchiveBytes: 256 * 1024 * 1024, + maxEntries: 50_000, + maxExtractedBytes: 512 * 1024 * 1024, + maxEntryBytes: 256 * 1024 * 1024, + maxMetaEntryBytes: 1024 * 1024, + maxEntryPathComponents: 64, + }, +}); +``` + +Returning `"skip"` rejects the archive unless `onFiltered: "skip-entry"` is +explicit. Zstd and bzip2 TAR are native-only; ZIP, TAR, and gzip retain guarded +JavaScript implementations. Catch `ArchiveLimitError` by its code, including +`archive-entry-path-components-exceeds-limit` for deep implicit-directory +attacks. See [Archive extraction](archive.md). + +## 4. Pick a publication failure policy + +`publishFileExclusive()` never replaces an existing target. Choose a strategy +and decide what a post-create directory-sync failure means to your application: + +```ts +await publishFileExclusive({ + sourcePath: stagedArchive, + targetPath: finalArchive, + strategy: "link-or-copy", + onSyncFailure: "preserve", +}); +``` + +`rollback` is the default: an unchanged target created by this call is removed +when directory sync throws. `preserve` keeps a complete but possibly +non-durable target and reports `cleanup: "preserved"` plus +`directorySync: { status: "failed", code? }` in the typed error. Backup +archives commonly need `preserve`; transactional protocols that expose only +durably committed names usually want `rollback`. See +[Directory durability](durability.md). + +## 5. Replace recursive scans with an explicit walk policy + +Use `Root.walk()` for caller-controlled relative paths. Every examined entry +consumes the budget even when filtered: + +```ts +for await (const entry of workspace.walk("memory", { + maxDepth: 12, + maxEntries: 50_000, + symlinkPolicy: "skip", + entryFilter: (entry) => + entry.kind === "directory" && entry.relativePath.endsWith("/.git") + ? "skip-subtree" + : "include", + onDirectoryError: "skip-and-report", +})) { + if (entry.kind === "directory-error") { + reportIncompleteSubtree(entry.relativePath, entry.error); + continue; + } + indexEntry(entry); +} +``` + +The default directory-error policy remains `throw`. See +[Directory walking](walk.md). + +## 6. Adopt the focused concurrency and secret APIs + +- Use `acquireFileLockSync()` only in synchronous boot or migration code; retry + waits block the thread. Request-serving paths should use `withFileLock()`. +- Use `createSecretFileAtomic()` for first-writer-wins credentials and catch + `secret-exists`; use `writeSecretFileAtomic()` only when replacement is the + intended protocol. +- Async `readSecretFile()` is strict; `tryReadSecretFile()` returns `undefined` + only for missing or blank content and still rejects suspicious files. +- Check `tempWorkspace.cleanup()` results when ownership matters; + `identity-mismatch` deliberately preserves a replacement path. + +See [File locks](sidecar-lock.md), [Secret files](secret-file.md), and +[Temp workspaces](temp.md). + +## 7. Gate native-only features + +`createPrivateDirectory()` is Windows-only and native-only because a pathname +fallback cannot promise the same creation-time DACL. Zstd/bzip2 extraction and +`strategy: "rename-noreplace"` are also native-only. Test the unavailable path +instead of assuming installation always succeeds. + +## 8. Run both behavior families in CI + +For each consumer workflow that matters: + +1. Run once with `FS_SAFE_NATIVE_MODE=auto` on every supported OS. +2. Run once with `FS_SAFE_NATIVE_MODE=off` to prove the JavaScript fallback. +3. Run native-required or native-only cases with `FS_SAFE_NATIVE_MODE=require`. +4. Exercise archive traversal/link/depth limits, publication sync failure, and + partial-walk reporting with production-shaped fixtures. + +For downstream staging and backup consumers: + +- [ ] Replace private whole-file hashing with `sha256File(path | FileHandle)` + from `durability`; native mode keeps digest work off the event loop and + the JavaScript fallback remains streaming. +- [ ] If Windows trust policy depends on exact principals, consume + `readOwnerAndDacl()` from `permissions`, reject incomplete/null/remote + descriptors as your policy requires, skip inherit-only ACEs where + appropriate, and apply the application's own SID allowlist. + +The [Testing](testing.md) page documents the test hooks and mode setup used by +fs-safe itself. diff --git a/docs/native-helper.md b/docs/native-helper.md new file mode 100644 index 0000000..0580c45 --- /dev/null +++ b/docs/native-helper.md @@ -0,0 +1,80 @@ +--- +title: Native helper policy +description: "How fs-safe loads its optional native filesystem primitives and how auto, require, and off affect guarded fallbacks." +--- + +# Native helper policy + +`@openclaw/fs-safe` installs `@openclaw/fs-safe-native` as an optional dependency. Its loader selects one of seven prebuilt packages for Linux x64/arm64 (glibc or musl), macOS x64/arm64, or Windows x64. Consumers do not compile Rust during installation. + +```ts +import { configureFsSafeNative } from "@openclaw/fs-safe/config"; + +configureFsSafeNative({ mode: "auto" }); // default +configureFsSafeNative({ mode: "off" }); // guarded JavaScript only +configureFsSafeNative({ mode: "require" }); // fail closed when the binding is unavailable +``` + +The equivalent environment variables are `FS_SAFE_NATIVE_MODE` and `OPENCLAW_FS_SAFE_NATIVE_MODE`. Accepted values are `auto`, `off`, `require`, `true`, `false`, `on`, `never`, `required`, `1`, and `0`. + +## Modes + +| Mode | Behavior | +|---|---| +| `auto` | Prefer native primitives when the current platform package loads; otherwise use the guarded JavaScript path. | +| `off` | Do not load a platform package. Use the guarded JavaScript path deterministically. | +| `require` | Throw `FsSafeError("helper-unavailable")` instead of falling back when an operation needs the native binding and it cannot load. | + +Configure the mode once during startup. Loading is lazy and cached; changing from `auto` to `require` after a failed load changes failure policy but does not repeatedly probe the package. + +## Native boundary + +The native package exposes policy-free filesystem mechanisms: beneath-root +open/mkdir/link, no-replace rename, identity reads, archive decode/execution, +clone/copy/hash workers, and Windows security descriptor calls. The TypeScript +layer owns policy, retries, filters, budgets, modes, cleanup, error +normalization, and the decision to fall back. + +- Linux uses `openat2` with `RESOLVE_BENEATH | RESOLVE_NO_MAGICLINKS` and `renameat2(RENAME_NOREPLACE)`. +- macOS resolves components with `O_NOFOLLOW`, restarts in-root symlinks from the pinned root descriptor, and uses `renameatx_np(RENAME_EXCL)`. +- Windows uses handle-relative `NtCreateFile`, rejects reparse points, and uses `FileRenameInfoEx` with replacement disabled. + +Native primitives back create-only pinned writes, async sidecar creation, +guarded publication, archive acceleration, and direct Windows ACL operations. +Equivalent JavaScript paths remain available for documented fallback-capable +features. See [Native architecture](native.md#javascript-fallback-guarantees-and-delta) +for the exact difference. + +## Migration from the Python helper + +Version 0.5 removes the Python worker and interpreter-path selection. The mode +contract is unchanged, so migrate startup configuration directly: + +| Python helper configuration | Native replacement | +|---|---| +| `configureFsSafePython({ mode: "auto" })` | `configureFsSafeNative({ mode: "auto" })` | +| `configureFsSafePython({ mode: "off" })` | `configureFsSafeNative({ mode: "off" })` | +| `configureFsSafePython({ mode: "require" })` | `configureFsSafeNative({ mode: "require" })` | +| `FS_SAFE_PYTHON_MODE` | `FS_SAFE_NATIVE_MODE` | +| `OPENCLAW_FS_SAFE_PYTHON_MODE` | `OPENCLAW_FS_SAFE_NATIVE_MODE` | +| `pythonPath`, `FS_SAFE_PYTHON`, and the OpenClaw interpreter-path aliases | Remove; prebuilt bindings do not use an interpreter path | + +In 0.5, `configureFsSafePython` and the legacy Python environment names +remain only as an upgrade bridge. On the first config read they emit one +`DeprecationWarning` with code `FS_SAFE_PYTHON_DEPRECATED`, state the mapped +native mode, and then apply that mode. A legacy interpreter path without an +explicit mode maps to `auto` and the path itself is ignored. Native config has +the normal precedence over legacy environment config. + +There is no silent alias and no Python execution fallback. The bridge exists +only to make shipped 0.4 configuration visible and predictable while the +consumer performs its 0.5 upgrade. + +## Related pages + +- [Config](config.md) +- [Security model](security-model.md) +- [Writing](writing.md) +- [File locks](sidecar-lock.md) +- [Durability](durability.md) +- [Migrating to 0.5](migrating-to-0.5.md) diff --git a/docs/native.md b/docs/native.md new file mode 100644 index 0000000..efd5428 --- /dev/null +++ b/docs/native.md @@ -0,0 +1,127 @@ +--- +title: Native architecture +description: "The optional native binding, fd-relative beneath model, platform mechanisms, loader security, and JavaScript fallback contract." +--- + +# Native architecture + +The optional `@openclaw/fs-safe-native` package supplies mechanisms that Node +does not expose directly. It is deliberately not a second policy engine. +TypeScript owns trusted-root selection, path validation, archive filtering, +budgets, modes, identity fencing, cleanup decisions, and error normalization. +Rust receives already-decided relative operations and performs the smallest +platform syscall sequence that can preserve the boundary. + +Every operation that has an equivalent safe Node implementation keeps that +guarded JavaScript path. Native loading is lazy and optional; installs do not +compile Rust. Native-only formats and creation-time Windows DACL guarantees +fail explicitly instead of substituting a weaker implementation. + +## The beneath model + +A trusted directory descriptor is the capability. Native operations accept +that descriptor plus a validated relative path and never reconstruct authority +from a process working directory. Newly created files use exclusive creation, +and TypeScript compares descriptor, pathname, and expected identities before +accepting results. + +Conceptually, a caller grants authority to an already-open root—not to a path +string that can be reinterpreted later: + +```text +validated Root handle + └─ relative components (untrusted) + └─ open/link/mkdir beneath the handle + └─ compare descriptor + pathname + expected identity +``` + +The TypeScript layer validates and decides. The native layer never decides +whether a path, archive entry, mode, owner, or cleanup policy is acceptable. + +- Linux uses `openat2(RESOLVE_BENEATH | RESOLVE_NO_MAGICLINKS)`, fd-relative + `mkdirat`/`linkat`/`renameat2`, `FICLONE`, and `copy_file_range`. +- macOS walks components with `openat(O_NOFOLLOW)`, restarts in-root symlinks + from the pinned root, uses `renameatx_np(RENAME_EXCL)`, and permits + `fclonefileat` in an owned, non-shared parent. The clone is normalized inside + a private staging directory: flags, ACLs, extended attributes, and broad mode + bits are cleared before no-replace publication. +- Windows uses handle-relative `NtCreateFile` with `OBJ_DONT_REPARSE` and + `FILE_OPEN_REPARSE_POINT`, then explicitly rejects reparse points. Rename and + hardlink operations stay rooted in already-open handles. Owner/DACL reads + use `GetSecurityInfo`; private directories receive their protected DACL in + the `CreateDirectoryW` call itself. + +## Archives + +Rust streams ZIP and TAR payloads, including gzip, zstd, and bzip2. It first +returns a bounded manifest. TypeScript applies the shared path, filter, strip, +mode, and byte policies and returns an index-bound extraction plan. Rust then +creates only those planned entries beneath a private staging descriptor. + +A fixed-512-byte pass-through meter sits between decompression and the TAR +crate. It reads only header type and octal/base-256 size fields. It never parses +metadata content. Oversized GNU long-name/link metadata is rejected before +buffering; PAX size overrides and GNU sparse entries are rejected as +unmeterable rather than guessed. The JavaScript node-tar path receives the same +`maxMetaEntryBytes` value and a matching fixed-header preflight. + +## Publication and hashing + +Exclusive publication tries a hardlink, then a copy-on-write clone, Linux +`copy_file_range`, and finally the existing asynchronous JavaScript byte loop. +All routes preserve `wx` semantics and the same source/target identity and +SHA-256 fencing. Native hashing and Linux whole-file copying run on N-API async +workers rather than the JavaScript event loop. + +## Mode semantics + +| Mode | Native loading | Fallback | +|---|---|---| +| `auto` | Try once, cache the result | Use guarded JavaScript when unavailable | +| `require` | Try once, cache the result | Throw `FsSafeError("helper-unavailable")` | +| `off` | Never attempt a binding load | Always use guarded JavaScript | + +The one exception is functionality with no safe JavaScript implementation: +zstd/bzip2 TAR and Windows private-directory creation fail with +`helper-unavailable` when native support is absent or off. + +## JavaScript fallback guarantees and delta + +Public policy does not change with the selected mechanism: traversal and link +rejection, archive filters/limits/modes, exclusive target creation, source and +target identity fencing, publication cleanup receipts, and secret/lock policy +remain TypeScript-owned. What changes is the syscall strength or availability: + +| Capability | Native path | Guarded JavaScript path | +|---|---|---| +| Root-relative opens/mutations | Descriptor-relative beneath operations; Linux uses `openat2`, Windows rejects reparse traversal in the object-manager call. | Lexical + canonical checks, no-follow opens where Node exposes them, private temp/rename, and post-operation identity verification. A hostile same-UID peer has a wider pathname race window. | +| ZIP/TAR/gzip | Rust streaming decode and fd-relative output creation. | JSZip/node-tar into a private stage, then the same guarded merge policy. | +| Zstd/bzip2 TAR | Supported. | Unsupported; typed `helper-unavailable`. | +| Publication copy | Clone, Linux `copy_file_range`, async native SHA-256. | Exclusive `wx` byte loop and Node SHA-256 with the same content/identity fences. | +| `rename-noreplace` | Atomic platform no-replace rename. | Unsupported; no emulation by check-then-rename. | +| Windows DACL read | Direct `GetSecurityInfo`; the public facts API exposes ordered basic allow/deny ACE SIDs, masks, and decoded flags without trust policy. | Established .NET/`icacls` inspection fallback for coarse permission checks; raw ACE facts are native-only. | +| Windows private directory | Creation-time protected DACL. | Unsupported; no weaker pathname-only substitute. | + +Use `off` in CI to keep the fallback contract exercised. Use `require` when a +deployment depends on the stronger mechanism or a native-only feature; do not +infer native loading from timing. + +## Loader security + +Importing fs-safe never executes a platform detector. Linux libc selection uses +the Node process report, conventional musl library filenames, and the ELF +`PT_INTERP` field of `process.execPath`. If all probes are inconclusive, the +loader conservatively attempts the glibc package and lets normal module loading +fail into `auto` fallback. After napi-rs generates bindings and declarations, +the hardening script replaces its broad platform/WASI template with the +checked-in seven-target loader this repository actually ships. Tests reject +`child_process`, `exec`, or `spawn` usage in the loader. + +## Related pages + +- [Native helper policy](native-helper.md) +- [Security model](security-model.md) +- [Archive extraction](archive.md) +- [Durability](durability.md) +- [Permissions](permissions.md) +- [Migrating to 0.5](migrating-to-0.5.md) diff --git a/docs/path.md b/docs/path.md index eb14713..45dc8ae 100644 --- a/docs/path.md +++ b/docs/path.md @@ -19,7 +19,7 @@ import { } from "@openclaw/fs-safe/path"; ``` -Only `root()`, `FsSafeError`, and the Python helper config live on the main entry. Path helpers are deliberately a subpath import so the main entry stays small. +Only `root()`, `FsSafeError`, and the native helper config live on the main entry. Path helpers are deliberately a subpath import so the main entry stays small. ## Boundary checks diff --git a/docs/permissions.md b/docs/permissions.md index 2eaf464..11434d2 100644 --- a/docs/permissions.md +++ b/docs/permissions.md @@ -64,7 +64,88 @@ createIcaclsResetCommand(targetPath, { isDir, env }); resolveWindowsUserPrincipal(env); ``` -The default Windows inspector calls `icacls.exe /sid` and classifies principals as trusted, world, or group. Trusted defaults include the current user, SYSTEM, and Administrators. The parser is on the advanced surface so tests and CLIs can process captured `icacls` output without spawning a process. +The fallback Windows inspector calls `icacls.exe ` using its supported +path-only inspection syntax and classifies principals as trusted, world, or +group. Trusted defaults include the current user, SYSTEM, and Administrators. +The parser is on the advanced surface so tests and CLIs can process captured +`icacls` output without spawning a process. + +When the optional native binding is available, `inspectPathPermissions()` +reads the owner and DACL directly with Windows security APIs. It classifies the +current user, LocalSystem, and built-in Administrators as trusted and reports +the world/group read/write facts consumed by secure reads. Descriptor forms it +cannot classify equivalently fall back to the established owner/.NET and +`icacls` path; `mode: "off"` exercises that fallback deterministically. + +## Policy-free owner and DACL facts + +`readOwnerAndDacl()` exposes the direct Windows descriptor facts needed by a +consumer that owns a principal allowlist. It deliberately does not decide +which SID is trusted or calculate effective access. For example, snapshot +staging can reject an incomplete descriptor and ignore inherit-only ACEs before +applying its own exact SID policy: + +```ts +import { readOwnerAndDacl } from "@openclaw/fs-safe/permissions"; + +const facts = readOwnerAndDacl(stagingDirectory); +if (facts.status === "unsupported-platform") { + throw new Error(`Windows ACL facts unavailable on ${facts.platform}`); +} +if (!facts.isLocal || !facts.daclPresent || !facts.complete) { + throw new Error("staging DACL cannot be evaluated completely"); +} + +for (const ace of facts.aces) { + if (ace.flags.inheritOnly) continue; + if (!trustedSids.has(ace.sid)) { + throw new Error(`unexpected staging principal: ${ace.sid}`); + } + evaluateMaskAndDenyOrder(ace.aceType, ace.mask); +} +``` + +On Windows the supported result contains `ownerSid`, `currentUserSid`, +`daclPresent`, `isLocal`, `complete`, `unsupportedAceTypes`, and ordered basic +allow/deny `aces`. `currentUserSid` is the process token's `TokenUser` SID, so +callers can compare it with the owner or their own allowlist without fs-safe +applying trust policy. Each ACE has `{ sid, mask, aceType, flags }`; `flags` +retains the raw byte and decoded +`objectInherit`, `containerInherit`, `noPropagateInherit`, `inheritOnly`, +`inherited`, `successfulAccess`, and `failedAccess` facts. SID strings are +lowercase Windows SID notation. `daclPresent: false` represents a null DACL, +which grants unrestricted access; it must not be mistaken for an empty DACL. + +Object-specific and other ACE layouts are not guessed: they are omitted, +`complete` becomes false, and their numeric types appear in +`unsupportedAceTypes`, allowing a security-sensitive caller to fail closed. +Non-Windows systems return `{ status: "unsupported-platform", platform }`. +Windows requires the optional native binding; if it is unavailable or forced +off, the call throws `FsSafeError("helper-unavailable")`. The existing coarse +`inspectPathPermissions()` API still owns its compatibility fallback and trust +classification. + +## Private directories + +```ts +import path from "node:path"; +import { createPrivateDirectory } from "@openclaw/fs-safe/permissions"; + +const sqliteDirectory = + "C:\\Users\\me\\AppData\\Local\\OpenClaw\\private-databases"; +await createPrivateDirectory(sqliteDirectory); +await openSqlite(path.join(sqliteDirectory, "sessions.sqlite")); +``` + +On Windows with native support, this creates the directory and applies a +protected owner + LocalSystem + Administrators full-control DACL directly with +an atomic security descriptor; no PowerShell or `icacls` process is launched. +This API is Windows-only and native-only; it fails closed with +`FsSafeError("helper-unavailable")` on other platforms, when native mode is off, +or when the binding is unavailable. POSIX callers should create private +directories through their existing trusted-root creation policy rather than a +pathname-only compatibility shim. Existing Windows permission inspection still +retains its .NET/`icacls` compatibility fallback. Use `createIcaclsResetCommand()` when you need a structured command and argv pair. Use `formatIcaclsResetCommand()` when you only need a remediation string for a user-facing message. @@ -96,3 +177,5 @@ type PermissionCheck = { - [Secure file reads](secure-file.md) — fd-pinned reads that enforce these checks. - [Errors](errors.md) — permission-related `FsSafeError` codes. +- [Native architecture](native.md) — direct Windows security descriptor mechanisms. +- [Migrating to 0.5](migrating-to-0.5.md) — native-only feature checklist. diff --git a/docs/python-helper.md b/docs/python-helper.md deleted file mode 100644 index 258b0eb..0000000 --- a/docs/python-helper.md +++ /dev/null @@ -1,103 +0,0 @@ ---- -title: Python helper policy -description: "How fs-safe uses its optional persistent Python helper, how to configure it, and what Node-only mode changes." ---- - -# Python helper policy - -`fs-safe` is a Node library. On POSIX systems, it can optionally keep one persistent Python helper process for filesystem operations that Node does not expose ergonomically as fd-relative APIs. - -The helper is not a sandbox and does not add new authority. It uses the same process user and the same filesystem permissions as your Node process. Its job is narrower: reduce race windows around parent-directory mutations after a root boundary has already been chosen. - -## Default - -The package default is: - -```ts -configureFsSafePython({ mode: "auto" }); -``` - -`auto` means: - -- use the helper for supported POSIX operations when it starts successfully; -- fall back to Node-only behavior when Python is missing, disabled by the host, or unavailable; -- keep the public API working in ordinary desktop, CI, Docker, and bundled-app environments. - -Applications can choose a stricter or simpler policy before the first filesystem operation: - -```ts -import { configureFsSafePython } from "@openclaw/fs-safe/config"; - -configureFsSafePython({ mode: "off" }); // never spawn Python -configureFsSafePython({ mode: "require" }); // fail closed if helper cannot start -``` - -Environment variables provide the same policy: - -```bash -FS_SAFE_PYTHON_MODE=auto # auto | off | require -FS_SAFE_PYTHON=/usr/bin/python3 -``` - -OpenClaw compatibility aliases are accepted too: `OPENCLAW_FS_SAFE_PYTHON_MODE`, `OPENCLAW_FS_SAFE_PYTHON`, `OPENCLAW_PINNED_PYTHON`, and `OPENCLAW_PINNED_WRITE_PYTHON`. - -## What the helper does - -Node's `fs` API is path-string oriented. It exposes `O_NOFOLLOW`, file handles, and some identity checks, but not a complete ergonomic `openat` / `renameat` / `unlinkat` / `mkdirat` surface for every operation `root()` needs. - -The helper fills that gap for supported POSIX operations: - -- stat/list paths relative to an already-open root directory; -- create directories while walking from a pinned parent; -- remove entries relative to a pinned parent; -- move entries with fd-relative rename semantics; -- run parent-fd write paths used by atomic replacement helpers. - -`fs-safe` sends requests to the helper over a JSON-lines protocol. It is one persistent process per Node process, not one Python spawn per filesystem call. - -## What you lose with `mode: "off"` - -Node-only mode still keeps the important application-level guardrails: - -- root-relative path validation; -- canonical root checks; -- no-follow opens where Node/platform support exists; -- file identity checks around reads and writes; -- atomic sibling-temp replacement; -- hardlink/symlink policy checks where the API requests them; -- byte limits and structured `FsSafeError` failures. - -What gets weaker is the POSIX defense against another same-UID process swapping a parent directory between validation and mutation. Without fd-relative mutation, `root().move()`, `root().remove()`, `root().mkdir()`, and some write paths rely on Node path operations plus pre/post checks instead of parent-fd syscalls. - -That is usually acceptable when the root directory is only writable by the trusted application user. It is not the right posture if untrusted local processes can race writes in the same tree and you are relying on `fs-safe` as part of the security boundary. - -## Choosing a mode - -| Mode | Use when | -|---|---| -| `auto` | You want the strongest available POSIX path when Python exists, but installs should still work without it. This is the package default. | -| `off` | You want deterministic Node-only behavior, no Python process, or a runtime that forbids spawning Python. | -| `require` | The fd-relative helper is part of your security posture and startup/runtime should fail closed if it is unavailable. | - -If you deploy with `require`, set `FS_SAFE_PYTHON` to an absolute interpreter path and test it in the same container, bundle, service manager, or sandbox that runs your app. - -## Application defaults - -Libraries should normally leave the package default alone. Applications can set a process-global policy once at startup: - -```ts -import { configureFsSafePython } from "@openclaw/fs-safe/config"; - -if (!process.env.FS_SAFE_PYTHON_MODE) { - configureFsSafePython({ mode: "off" }); -} -``` - -This is the right shape for apps that want to make Python an explicit operator choice while still letting deployment env vars opt back into `auto` or `require`. - -## Related pages - -- [Security model](security-model.md) — what `root()` does and does not promise. -- [Root API](root.md) — root-bounded read/write/move/remove methods. -- [Errors](errors.md) — `helper-unavailable` and `helper-failed` handling. -- [Testing](testing.md) — forcing helper modes in tests. diff --git a/docs/quickstart.md b/docs/quickstart.md index 157b655..c473472 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -121,7 +121,10 @@ await extractArchive({ maxEntries: 50_000, maxExtractedBytes: 512 * 1024 * 1024, maxEntryBytes: 256 * 1024 * 1024, + maxMetaEntryBytes: 1024 * 1024, + maxEntryPathComponents: 64, }, + entryModes: "clamp", }); ``` @@ -133,7 +136,7 @@ Extraction stages into a private dir and merges through the same boundary used b import { withTempWorkspace } from "@openclaw/fs-safe/temp"; await withTempWorkspace({ rootDir: "/srv/jobs/tmp", prefix: "build-" }, async (workspace) => { - await fs.copyIn("input.bin", "/tmp/source.bin"); + await workspace.copyIn("input.bin", "/tmp/source.bin"); // ...do work in workspace.dir; auto-cleaned on exit }); ``` diff --git a/docs/root.md b/docs/root.md index cea1781..7123bd5 100644 --- a/docs/root.md +++ b/docs/root.md @@ -51,8 +51,23 @@ fs.readJson(rel, options?) // parsed T fs.open(rel, options?) // { handle, realPath, stat, [Symbol.asyncDispose] } fs.readAbsolute(absPath, options?) // ReadResult; absPath must already be inside the root fs.reader(options?) // (path) => Promise; useful for loader APIs +fs.walk(rel, options) // root-bounded AsyncIterable<{ relativePath, kind, size }> ``` +`walk()` is the incremental, root-bounded recursive scan. It supports entry and +depth budgets, cancellation, and `symlinkPolicy: "skip" | +"follow-within-root"`. Budget exhaustion yields a `"truncated"` marker by +default or throws `FsSafeError("too-large")` with `limitBehavior: "throw"`. +Use `entryFilter(entry)` to return `"include"`, `"skip"`, or +`"skip-subtree"`. `"skip"` omits the current entry but still descends into a +directory; `"skip-subtree"` omits a directory and all of its descendants. +Directory reads remain fail-fast by default. With +`onDirectoryError: "skip-and-report"`, the iterator instead yields +`{ relativePath, kind: "directory-error", size: 0, error }` and continues with +the remaining tree. +See [Directory walking](walk.md) for the pure-Node guarantees and the contrast +with the standalone best-effort walkers. + `open()` returns a Node `FileHandle` for streaming. Prefer `await using` for cleanup: ```ts @@ -100,25 +115,22 @@ fs.resolve(rel) // absolute path inside the root, after canonic These do not pin a later operation. They are safe to expose to UIs and decision points; for the actual read or write, use the verb methods so the operation pins identity at the point of use. -## Python helper mode +## Native helper mode -On POSIX, mutation and inspection methods that need fd-relative directory -operations go through one persistent Python helper process. This avoids a -spawn-per-call cost while still using `openat`/`renameat`/`unlinkat`-style -operations that Node's `fs` API does not expose ergonomically. +Create-only writes prefer the optional native helper for fd-relative opens and +atomic no-replace rename. Operations without native wiring retain their guarded +JavaScript implementations. ```ts -import { configureFsSafePython } from "@openclaw/fs-safe/config"; +import { configureFsSafeNative } from "@openclaw/fs-safe/config"; -configureFsSafePython({ mode: "off" }); // Node-only fallback path -configureFsSafePython({ mode: "require" }); // fail if fd-relative helper unavailable +configureFsSafeNative({ mode: "off" }); // guarded JavaScript path +configureFsSafeNative({ mode: "require" }); // fail if the binding is unavailable ``` -`auto` is the default. Configure the mode before creating roots. Without the -helper, root methods still run, but same-UID races that swap parent directories -between validation and mutation are harder to close completely. Use `require` -when that downgrade should be treated as a deployment failure. See -[Python helper policy](python-helper.md) for deployment guidance. +`auto` is the default. Configure the mode before creating roots. See the +[native helper policy](native-helper.md) for supported platforms, the native +surface, and the precise fallback boundary. ### Properties diff --git a/docs/secret-file.md b/docs/secret-file.md index 53e4ee7..3b9c5e7 100644 --- a/docs/secret-file.md +++ b/docs/secret-file.md @@ -4,7 +4,10 @@ Helpers for reading and writing credentials. Files are written at mode `0o600`, ```ts import { + createSecretFileAtomic, + readSecretFile, readSecretFileSync, + tryReadSecretFile, tryReadSecretFileSync, writeSecretFileAtomic, DEFAULT_SECRET_FILE_MAX_BYTES, @@ -54,7 +57,7 @@ if (token) { Strict reader. Throws `FsSafeError` when the file is missing, too large, empty, unreadable, or rejected by the validation checks. Use when failing loudly is the right call: ```ts -const token = readSecretFileSync("/var/lib/app/auth.token"); +const token = readSecretFileSync("/var/lib/app/auth.token", "auth token"); ``` ### Read options @@ -69,6 +72,24 @@ type SecretFileReadOptions = { The reader trims the file content and rejects empty results. `rejectSymlink` blocks a symlink path before the pinned read. Hardlinks are rejected by default so another in-tree name cannot alias the credential; pass `rejectHardlinks: false` only when you explicitly trust that layout. +`readSecretFile()` and `tryReadSecretFile()` are asynchronous counterparts with +the same pinned-handle validation, byte cap, trimming, error codes, and strict +versus missing-is-undefined naming semantics. + +Use the async strict reader when a service cannot start safely without the +credential: + +```ts +import { readSecretFile } from "@openclaw/fs-safe/secret"; + +const signingKey = await readSecretFile( + "/var/lib/app/keys/webhook-signing.key", + "webhook signing key", + { maxBytes: 8 * 1024, rejectSymlink: true }, +); +startWebhookVerifier(signingKey); +``` + ## Writing ### `writeSecretFileAtomic(params)` @@ -99,6 +120,33 @@ type WriteSecretFileParams = { The directory mode is asserted on each component along the path: `rootDir`, then any intermediate dirs, then the parent. The helper enforces that every component matches `dirMode` — wider permissions on an existing directory cause the write to fail. Audit and tighten existing secret directories yourself. +### `createSecretFileAtomic(params)` + +This create-only sibling has the same directory, mode, pinned-write, and +post-write verification policy. Final materialization uses exclusive create; +if anything already occupies the target path it throws +`FsSafeError("secret-exists")` without modifying that entry. Use the distinct +name when first-writer-wins is part of the credential protocol. + +For example, two onboarding requests may race to install the first refresh +token. Exactly one should win, and the loser must not overwrite it: + +```ts +import { FsSafeError } from "@openclaw/fs-safe/errors"; +import { createSecretFileAtomic } from "@openclaw/fs-safe/secret"; + +try { + await createSecretFileAtomic({ + rootDir: "/var/lib/app/credentials", + filePath: "/var/lib/app/credentials/provider.refresh-token", + content: refreshToken, + }); +} catch (error) { + if (!(error instanceof FsSafeError) || error.code !== "secret-exists") throw error; + // Another initializer won. Read and validate the installed credential. +} +``` + For more permissive credentials, override `mode`: ```ts @@ -153,3 +201,4 @@ await withTimeout( - [JSON files](json.md) — `writeJson` accepts `mode: 0o600` for non-secret JSON state. - [Atomic writes](atomic.md) — the lower-level `replaceFileAtomic` used by these helpers. - [Private file-store mode](private-file-store.md) — root-bounded JSON+text stores using secret-file write policy. +- [Migrating to 0.5](migrating-to-0.5.md) — strict/try reads and create-only adoption checklist. diff --git a/docs/security-model.md b/docs/security-model.md index bac41d2..f6bb751 100644 --- a/docs/security-model.md +++ b/docs/security-model.md @@ -86,8 +86,9 @@ The library does not modify or constrain the global Node.js `fs` namespace, and ## Platform notes -- **POSIX (Linux, macOS):** Best-defended path. Uses `O_NOFOLLOW`, fd identity checks, and one persistent Python helper process for fd-relative `unlinkat` / `mkdirat` / `renameat` / parent-fd write operations. Configure `FS_SAFE_PYTHON_MODE=require` when helper startup must fail closed, or `off` when you need a no-Python runtime. See [Python helper policy](python-helper.md). -- **Windows:** Falls back to the safest Node-level behavior available. `O_NOFOLLOW` is not honored. Some fd-relative POSIX hardening is unavailable. The library does the path canonicalization, identity, and atomic-rename checks it can. +- **Linux:** Native opens use `openat2(RESOLVE_BENEATH | RESOLVE_NO_MAGICLINKS)` and no-replace publication uses `renameat2(RENAME_NOREPLACE)`; guarded JavaScript implementations remain for operations outside the native surface. +- **macOS:** Native opens walk with `O_NOFOLLOW` and re-resolve in-root symlinks from the pinned root descriptor; no-replace publication uses `renameatx_np(RENAME_EXCL)`. +- **Windows:** Native opens are handle-relative and reject reparse points; no-replace publication uses `FileRenameInfoEx` with replacement disabled. Other operations use the guarded Node implementation. The library does not advertise different security guarantees per platform — it advertises the same surface and relies on the strongest mechanism the platform offers. @@ -102,7 +103,7 @@ The library does not advertise different security guarantees per platform — it | Hardlink rejection is best-effort | Link-count checks depend on platform metadata. Treat `hardlinks: "reject"` as a tripwire, not an authorization primitive. | | Mode bits are not a full policy engine | `replaceFileAtomic` and secret-file helpers set requested modes, but you should still set umask and inspect modes when policy requires it. | | Archive extraction is path safety, not content safety | Unsafe entry paths and links are rejected; malicious payload contents remain your application layer's problem. | -| Helper failures degrade fd-relative hardening | `helper-unavailable` falls back in `auto` mode and fails closed in `require` mode. Atomicity and identity checks remain, but parent-directory swaps between validation and mutation are less tightly pinned without the helper. | +| Native package unavailable | `helper-unavailable` falls back in `auto` mode and fails closed in `require` mode for native-backed operations. Guarded JavaScript atomicity and identity checks remain. | | FUSE mounts with rename-unstable inode numbers | Some FUSE mounts (rclone is a confirmed example) do not preserve source inode identity at the rename destination. The explicit `renameIdentity: "verify-content-with-lock"` compatibility mode verifies content under a cooperative lock for that boundary only; subsequent path identity checks and the default remain strict. See [Writing](writing.md) for the weaker opt-in contract. | ## Recommended deployment shape diff --git a/docs/sidecar-lock.md b/docs/sidecar-lock.md index 7d6cf00..29776a2 100644 --- a/docs/sidecar-lock.md +++ b/docs/sidecar-lock.md @@ -42,6 +42,9 @@ function withFileLock( ): Promise; function createFileLockManager(key: string): FileLockManager; + +function acquireFileLockSync(targetPath: string, options: FileLockSyncAcquireOptions): FileLockSyncHandle; +function withFileLockSync(targetPath: string, options: FileLockSyncAcquireOptions, fn: () => T): T; ``` `managerKey` is an optional identifier used to keep state isolated across multiple lock domains in the same process. Use distinct keys for distinct domains (`"snapshot"`, `"compact"`, `"build"`). If omitted, fs-safe derives one from the target path. @@ -73,11 +76,15 @@ type FileLockAcquireOptions> = { payload: Record | null; }) => boolean | Promise; metadata?: Record; // attached to heldEntries() output for diagnostics + parsePayload?: (raw: string) => unknown; + lockRoot?: Root; + onCompromised?: (info: { lockPath: string; normalizedTargetPath: string }) => void; + compromiseCheckIntervalMs?: number; }; type FileLockRetryOptions = { retries?: number; // number of retry attempts after the first failure - factor?: number; // exponential backoff factor (default 2) + factor?: number; // exponential backoff factor (default 1: constant delay) minTimeout?: number; // initial delay (ms) maxTimeout?: number; // delay cap (ms) randomize?: boolean; // jitter @@ -85,6 +92,14 @@ type FileLockRetryOptions = { ``` `payload` is a function so you can re-evaluate it on each retry (e.g. timestamp, PID). +`parsePayload` replaces JSON parsing for legacy or custom sidecars. Its `unknown` +result is passed to `shouldReclaim` and `shouldRemoveStaleLock`, allowing PID, +process-start, argv, or role schemas to remain application-owned. + +Pass `lockRoot` to place sidecar create, read, verification, and removal behind +an existing `Root` capability. `lockPath` must resolve inside that root. +Identity-conditioned removal remains the only release and reclaim deletion +path. ## Release handle @@ -92,25 +107,49 @@ type FileLockRetryOptions = { type FileLockHandle = { lockPath: string; normalizedTargetPath: string; + verifyStillHeld: () => Promise; release: () => Promise; [Symbol.asyncDispose](): Promise; }; ``` +`verifyStillHeld()` compares the current sidecar with the ownership snapshot +captured at acquisition. Set `compromiseCheckIntervalMs` together with +`onCompromised` for a cheap periodic check; the callback fires once after the +sidecar no longer matches. This is detection, not revocation of work already in +progress. + +## Synchronous locks + +`acquireFileLockSync()` and `withFileLockSync()` mirror filesystem arbitration, +retry, payload parsing, stale policy, guarded identity-conditioned reclaim, +verification, and compromise monitoring. They do not use the async manager +queue, support async callbacks, or provide same-process reentrancy. Retry waits +block the calling thread; use the async API in request-serving code. + Always release in a `finally`: ```ts -const handle = await acquireFileLock(targetPath, { +import { acquireFileLockSync } from "@openclaw/fs-safe/file-lock"; + +const handle = acquireFileLockSync("/var/lib/app/schema.json", { staleMs: 60_000, - payload: () => ({ pid: process.pid }), + timeoutMs: 5_000, + retry: { retries: 20, minTimeout: 25, maxTimeout: 250 }, + payload: () => ({ pid: process.pid, operation: "schema-migration" }), }); try { - await doExclusiveWork(); + if (!handle.verifyStillHeld()) throw new Error("migration lock was replaced"); + migrateSchemaSynchronously(); } finally { - await handle.release(); + handle.release(); } ``` +The sync payload, reclaim, and parsing callbacks must also be synchronous. This +shape is appropriate for a short boot migration; it is a poor fit for a server +request because retry backoff uses a blocking wait. + If your process dies before `release()` runs and skips the exit handler, the sidecar remains. Once `staleMs` elapses (or your `shouldReclaim` returns true), acquisition fails closed by default instead of deleting by path. ## `withFileLock` — common shape made one-liner @@ -248,3 +287,4 @@ await withFileLock( - [Atomic writes](atomic.md) — single-writer atomicity that often replaces the need for a lock entirely. - `createAsyncLock` from `@openclaw/fs-safe/advanced` — in-process serialization for a single Node process. +- [Migrating to 0.5](migrating-to-0.5.md) — choosing sync versus async lock APIs. diff --git a/docs/temp.md b/docs/temp.md index 4b5f993..5db88b8 100644 --- a/docs/temp.md +++ b/docs/temp.md @@ -23,6 +23,7 @@ The compact factory. Returns: ```ts type TempWorkspace = { dir: string; + identity: { dev: number | bigint; ino: number | bigint }; store: FileStore; path(fileName: string): string; write(fileName: string, data: string | Uint8Array): Promise; @@ -30,7 +31,7 @@ type TempWorkspace = { writeJson(fileName: string, data: unknown, options?: { trailingNewline?: boolean }): Promise; copyIn(fileName: string, sourcePath: string): Promise; read(fileName: string): Promise; - cleanup(): Promise; + cleanup(): Promise<"removed" | "missing" | "identity-mismatch">; [Symbol.asyncDispose](): Promise; }; ``` @@ -58,6 +59,28 @@ await state.write({ ready: true }); The workspace owns cleanup; the store is only a view over the workspace directory. +The identity receipt is captured when the workspace is created. Manual, +disposal, and process-exit cleanup remove the path only while `lstat` still +matches that receipt. If another actor renames the workspace away and places a +new directory at the old name, cleanup returns `"identity-mismatch"` and leaves +the replacement untouched. Disposal hooks perform the same check and ignore +the returned status. + +When cleanup is part of a retention or audit decision, inspect the receipt +instead of treating cleanup as fire-and-forget: + +```ts +const workspace = await tempWorkspace({ rootDir: "/var/lib/app/tmp", prefix: "restore-" }); +try { + await restoreInto(workspace.dir); +} finally { + const cleanup = await workspace.cleanup(); + if (cleanup === "identity-mismatch") { + alertOperator("restore workspace path was replaced; replacement preserved"); + } +} +``` + The sync variant `tempWorkspaceSync` exposes the same surface with sync return types and a `FileStoreSync` at `workspace.store`. diff --git a/docs/test-hooks.md b/docs/test-hooks.md index f4e2fdd..dbb0dc7 100644 --- a/docs/test-hooks.md +++ b/docs/test-hooks.md @@ -31,6 +31,15 @@ type FsSafeTestHooks = { afterPreOpenLstat?: (filePath: string) => Promise | void; beforeOpen?: (filePath: string, flags: number) => Promise | void; afterOpen?: (filePath: string, handle: FileHandle) => Promise | void; + beforeArchiveOutputMutation?: (operation: "mkdir" | "chmod", targetPath: string) => Promise | void; + beforeFileStorePruneDescend?: (dirPath: string) => Promise | void; + beforeFileStoreSyncPrivateWrite?: (filePath: string) => void; + beforeRootFallbackMutation?: (operation: "mkdir" | "move" | "remove", targetPath: string) => Promise | void; + afterPinnedWriteFallbackRename?: (targetPath: string) => Promise | void; + beforeSiblingTempWrite?: (tempPath: string) => Promise | void; + beforeTrashMove?: (targetPath: string, destPath: string) => void; + afterPublishTargetCreated?: (method, targetPath, identity) => Promise | void; + beforePublishDirectorySync?: (method, targetPath, identity) => Promise | void; }; ``` @@ -39,8 +48,19 @@ type FsSafeTestHooks = { | `afterPreOpenLstat` | A pre-open `lstat` has just resolved. Use this to swap a path between validation and open. | | `beforeOpen` | The library is about to call `open(path, flags)`. Use this to inject a TOCTOU window. | | `afterOpen` | An open just succeeded. Use this to mutate state before the post-open identity check runs. | - -Each hook may be sync or async; async hooks are awaited. +| `beforeArchiveOutputMutation` | Archive staging is about to create a directory or apply a mode. | +| `beforeFileStorePruneDescend` | File-store pruning is about to descend into a directory. | +| `beforeFileStoreSyncPrivateWrite` | A synchronous private-store write is about to mutate its target. | +| `beforeRootFallbackMutation` | A guarded JS root fallback is about to mkdir, move, or remove. | +| `afterPinnedWriteFallbackRename` | A fallback rename committed and post-commit identity checks have not run yet. | +| `beforeSiblingTempWrite` | A sibling temp file exists and its writer is about to run. | +| `beforeTrashMove` | Trash handling is about to move the target. | +| `afterPublishTargetCreated` | Exclusive publication created its target and final fences have not run yet. | +| `beforePublishDirectorySync` | Publication verified the target and is about to sync its parent directory. | + +Hooks typed `Promise | void` may be sync or async and are awaited. +Hooks used by synchronous code paths are typed `void` and must not return a +promise. ## Usage diff --git a/docs/testing.md b/docs/testing.md index e858561..96c6419 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1,6 +1,9 @@ # Testing -`@openclaw/fs-safe/test-hooks` exposes a small set of test-only injection points. They are inert in production: the hooks only activate when `process.env.NODE_ENV === "test"`. Outside test mode, calls to set hooks are no-ops, so leaking a test setup line into production is safe but ineffective. +`@openclaw/fs-safe/test-hooks` exposes test-only injection points. Registration +is allowed only when `process.env.NODE_ENV === "test"` or +`process.env.VITEST === "true"`; registering a non-empty hook set elsewhere +throws. Production code must not import this subpath. ```ts import { @@ -14,7 +17,7 @@ The double-underscore prefix is a deliberate "hands off" signal: production code ## When to reach for hooks - Reproduce a TOCTOU race deterministically: simulate a symlink swap between resolve and open, or between write and rename. -- Force Node-only behavior without uninstalling Python from your runners. +- Force guarded JavaScript behavior without removing native packages from your runners. - Inject latency to test cancellation/timeout paths. If you don't need to inject a race, you don't need hooks — most tests should drive the library through normal calls and assert on observable behavior. @@ -26,6 +29,10 @@ type FsSafeTestHooks = { afterPreOpenLstat?: (filePath: string) => Promise | void; beforeOpen?: (filePath: string, flags: number) => Promise | void; afterOpen?: (filePath: string, handle: import("node:fs/promises").FileHandle) => Promise | void; + afterPublishTargetCreated?: (method, targetPath, identity) => Promise | void; + beforePublishDirectorySync?: (method, targetPath, identity) => Promise | void; + // Additional archive, store, root-fallback, temp, and trash race hooks are + // documented on the focused Test hooks reference page. }; function __setFsSafeTestHooksForTest(hooks?: FsSafeTestHooks): void; @@ -37,6 +44,8 @@ Hooks are called at well-defined points in the library's hot paths: - **`afterPreOpenLstat`** — runs after the pre-open `lstat`. A common use is to swap the path's target via `fs.symlink`/`fs.unlink` to drive a TOCTOU race. - **`beforeOpen`** — runs before `fs.open` with the exact flags the root read path will use. - **`afterOpen`** — runs after the file handle is opened. Useful to wrap handle methods or inject a size race before a stream is consumed. +- **`afterPublishTargetCreated`** — runs after exclusive publication created a target but before its final fences. +- **`beforePublishDirectorySync`** — runs after target verification and immediately before strict parent sync; useful for exercising `onSyncFailure`. `__setFsSafeTestHooksForTest(undefined)` clears all hooks. Always clean up between tests. @@ -81,20 +90,20 @@ it("rejects a swap between resolve and open", async () => { The `code` may be `symlink` (caught at open by `O_NOFOLLOW`) or `path-mismatch` (caught by the post-open identity check) depending on platform — both are correct refusals. -## Example: force Node-only fallback behavior +## Example: force guarded JavaScript fallback behavior ```ts -import { configureFsSafePython } from "@openclaw/fs-safe/config"; +import { configureFsSafeNative } from "@openclaw/fs-safe/config"; beforeEach(() => { - configureFsSafePython({ mode: "off" }); + configureFsSafeNative({ mode: "off" }); }); afterEach(() => { - configureFsSafePython({ mode: "auto", pythonPath: undefined }); + configureFsSafeNative({ mode: "auto" }); }); -it("runs without the Python helper", async () => { +it("runs without the native helper", async () => { const fs = await root(dir); await fs.write("file.txt", "ok"); await expect(fs.readText("file.txt")).resolves.toBe("ok"); @@ -116,6 +125,8 @@ afterEach(() => { A global hook clear in your test setup file is a good safety net. +See the [complete Test hooks reference](test-hooks.md) for every optional hook. + ## Patterns for testing fs-safe-using code You usually don't need hooks. Most tests follow this shape: diff --git a/docs/walk.md b/docs/walk.md index 57792ba..761c882 100644 --- a/docs/walk.md +++ b/docs/walk.md @@ -67,7 +67,62 @@ type WalkDirectoryOptions = { Unreadable directories are skipped rather than throwing, but every skipped directory is recorded in `failedDirs`. This keeps the helper suitable for best-effort inventories while letting pruning jobs tell an incomplete scan from an empty one: a destructive reconcile that deletes state for paths missing from `entries` must first confirm `failedDirs` holds no real read failures, or a transient `EIO`/`EACCES` blip would be mistaken for mass deletion. Use a stricter root-bounded operation when every entry must be accounted for. +## Root-bounded async iteration + +`Root.walk(rel, options)` is the root-bounded counterpart to these standalone +inventory helpers. It yields `{ relativePath, kind, size }` incrementally and +accepts `maxDepth`, `maxEntries`, `symlinkPolicy: "skip" | +"follow-within-root"`, and an `AbortSignal`. The default budget behavior yields +one `kind: "truncated"` marker and ends; pass `limitBehavior: "throw"` for a +typed `FsSafeError("too-large")` instead. + +`entryFilter` is evaluated for each resolved file, directory, or other entry: + +```ts +for await (const entry of capability.walk("", { + symlinkPolicy: "skip", + entryFilter: (entry) => + entry.kind === "directory" && entry.relativePath === ".git" + ? "skip-subtree" + : "include", + onDirectoryError: "skip-and-report", +})) { + if (entry.kind === "directory-error") { + console.warn("incomplete subtree", entry.relativePath, entry.error); + continue; + } + consume(entry); +} +``` + +The result values are `"include"`, `"skip"`, and `"skip-subtree"`. Plain +`"skip"` omits an entry but still descends when it is a directory; +`"skip-subtree"` omits that directory and prunes its descendants. Returning +`"skip-subtree"` for a non-directory is equivalent to `"skip"`. + +`onDirectoryError` defaults to `"throw"`, preserving the original fail-fast +contract. `"skip-and-report"` yields a discriminated +`{ kind: "directory-error", relativePath, size: 0, error }` marker for a +directory that cannot be resolved or listed, then continues with its siblings. +Every examined directory entry consumes `maxEntries` before filtering, so +`"skip"` cannot turn the iterator into an unbounded traversal. Reporting and +`"truncated"` markers describe already-reached state and do not authorize +further descent. + +The pure-Node path validates every directory canonically inside the root, +revalidates each listing through the normal `Root.list()` boundary, and tracks +canonical directories to stop symlink cycles. It does not hold a descriptor +for the entire tree, so it is not a process sandbox against a hostile peer that +can continuously swap and restore directories. Each individual lookup retains +the documented Node `Root` boundary checks. + +Unlike `walkDirectory()` and `walkDirectorySync()`, `Root.walk()` is +root-bounded and reports failures inline because an async iterator has no final +result summary. Its default remains to throw on unreadable or invalid +directories. + ## See also - [`fileStore`](file-store.md) — managed stores use bounded walking for pruning. - [Path scopes](path-scope.md) — boundary checks for known absolute paths. +- [Migrating to 0.5](migrating-to-0.5.md) — adopting bounded pruning and partial-result handling. diff --git a/docs/writing.md b/docs/writing.md index f08448e..b94769e 100644 --- a/docs/writing.md +++ b/docs/writing.md @@ -246,9 +246,9 @@ const fs = await root("/mnt/rclone-workspace", { await fs.write("state.json", body); // succeeds on rclone FUSE ``` -**How it works.** The full write runs under an exclusive per-target lock named `.fs-safe-write-.lock` in the root. Keeping the lock in the already-canonical root avoids creating an unguarded lock path through a missing or raced target parent. The guarded Node fallback accepts the source-temp-to-destination inode mismatch only when the SHA-256 of the re-read bytes matches the SHA-256 of the bytes written. Subsequent path identity checks remain strict, so this mode requires an unchanged destination path to report stable identity. It deliberately bypasses the stricter fd-relative Python helper because that helper requires rename to preserve inode identity. The lock is released before the call returns. +**How it works.** The full write runs under an exclusive per-target lock named `.fs-safe-write-.lock` in the root. Keeping the lock in the already-canonical root avoids creating an unguarded lock path through a missing or raced target parent. The guarded Node fallback accepts the source-temp-to-destination inode mismatch only when the SHA-256 of the re-read bytes matches the SHA-256 of the bytes written. Subsequent path identity checks remain strict, so this mode requires an unchanged destination path to report stable identity. It deliberately stays on the guarded JavaScript path because content verification replaces the normal inode-preserving rename contract. The lock is released before the call returns. -**Security note.** `verify-content-with-lock` proves that the bytes observed after rename match the requested write and prevents *cooperating* writers from interleaving. It does **not** prove that the destination still names the temp-file object, retain the Python helper's fd-relative parent pinning, or stop a same-UID process that ignores the advisory lock. Do not use this option on directories writable by untrusted same-UID processes. Strict identity verification remains the default. +**Security note.** `verify-content-with-lock` proves that the bytes observed after rename match the requested write and prevents *cooperating* writers from interleaving. It does **not** prove that the destination still names the temp-file object, retain fd-relative parent pinning, or stop a same-UID process that ignores the advisory lock. Do not use this option on directories writable by untrusted same-UID processes. Strict identity verification remains the default. Lock recovery is fail-closed. If a process crashes and leaves the root-level `.fs-safe-write-.lock`, a later write reports the stale lock instead of deleting it based on a host-local PID. Recover only under external authority that excludes every competing writer; see [File lock](sidecar-lock.md#stale-recovery-guarded-remove-if-unchanged). diff --git a/native/Cargo.toml b/native/Cargo.toml new file mode 100644 index 0000000..51616ca --- /dev/null +++ b/native/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "fs-safe-native" +version = "0.5.0-dev.0" +edition = "2024" +license = "MIT" +repository = "https://github.com/openclaw/fs-safe" +rust-version = "1.85" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +bzip2 = "0.6.1" +flate2 = "1.1.9" +napi = { version = "3.11.0", default-features = false, features = ["dyn-symbols", "napi8"] } +napi-derive = "3.6.0" +sha2 = "0.11.0" +tar = { version = "0.4.46", default-features = false } +zip = { version = "6.0.0", default-features = false, features = ["deflate-flate2"] } +zstd = { version = "0.13.3", default-features = false } + +[target.'cfg(target_os = "linux")'.dependencies] +rustix = { version = "1.1.4", features = ["fs"] } + +[target.'cfg(target_os = "macos")'.dependencies] +libc = "0.2.177" +rustix = { version = "1.1.4", features = ["fs"] } + +[target.'cfg(target_os = "windows")'.dependencies] +windows-sys = { version = "0.61.2", features = [ + "Win32_Foundation", + "Win32_Security", + "Win32_Security_Authorization", + "Win32_Storage_FileSystem", + "Win32_System_IO", + "Win32_System_LibraryLoader", + "Win32_System_Threading", +] } + +[build-dependencies] +napi-build = "2.2.4" diff --git a/native/build.rs b/native/build.rs new file mode 100644 index 0000000..0f1b010 --- /dev/null +++ b/native/build.rs @@ -0,0 +1,3 @@ +fn main() { + napi_build::setup(); +} diff --git a/native/index.d.ts b/native/index.d.ts new file mode 100644 index 0000000..0a3bfdc --- /dev/null +++ b/native/index.d.ts @@ -0,0 +1,100 @@ +/* auto-generated by NAPI-RS */ +/* eslint-disable */ +export declare function cloneFileExclusive(sourceFd: number, targetRootFd: number, targetRelPath: string): number + +export declare function copyFileRangeExclusive(sourceFd: number, targetRootFd: number, targetRelPath: string): Promise + +export declare function createPrivateDirectory(path: string): void + +export declare function extractArchiveNative(path: string, kind: string, rootFd: number, plan: Array, maxMetaEntryBytes: number, signal: AbortSignal): Promise + +export interface FileHash { + bytes: number + digest: string +} + +export interface FileIdentity { + dev: number + ino: number + mode: number + nlink: number + size: number + isFile: boolean + isDirectory: boolean + isSymbolicLink: boolean +} + +export declare function fstatIdentity(fd: number): FileIdentity + +export declare function inspectArchiveNative(path: string, kind: string, maxEntries: number, maxMetaEntryBytes: number, maxManifestBytes: number, signal: AbortSignal): Promise> + +export declare function linkBeneath(sourceRootFd: number, sourceRelPath: string, targetRootFd: number, targetRelPath: string): void + +export declare function mkdirBeneath(rootFd: number, relPath: string, mode: number): void + +export interface NativeArchiveEntry { + index: number + path: string + kind: string + size: number + mode: number +} + +export interface NativeArchivePlanEntry { + index: number + path: string + kind: string + size: number + mode: number +} + +export interface NativeCopyResult { + fd: number + bytes: number + errorCode?: string + errorMessage?: string +} + +export declare function openBeneath(rootFd: number, relPath: string, flags: number): number + +export declare function readArchiveEntryNative(path: string, kind: string, requested: string, maxBytes: number, maxEntries: number, maxMetaEntryBytes: number, signal: AbortSignal): Promise + +export declare function readOwnerAndDacl(path: string): WindowsSecurityFacts + +export declare function renameNoReplace(sourceRootFd: number, sourceRelPath: string, targetRootFd: number, targetRelPath: string): void + +export declare function sha256File(fd: number): Promise + +export interface WindowsAccessControlEntry { + sid: string + mask: number + aceType: string + flags: WindowsAceFlags +} + +export interface WindowsAceFlags { + raw: number + objectInherit: boolean + containerInherit: boolean + noPropagateInherit: boolean + inheritOnly: boolean + inherited: boolean + successfulAccess: boolean + failedAccess: boolean +} + +export interface WindowsSecurityFacts { + ownerSid: string + currentUserSid: string + ownerClass: string + worldWritable: boolean + groupWritable: boolean + worldReadable: boolean + groupReadable: boolean + fallbackRequired: boolean + daclPresent: boolean + isLocal: boolean + aceListComplete: boolean + unsupportedAceTypes: Array + aces: Array +} diff --git a/native/index.js b/native/index.js new file mode 100644 index 0000000..687acf4 --- /dev/null +++ b/native/index.js @@ -0,0 +1,179 @@ +// prettier-ignore +/* eslint-disable */ +// @ts-nocheck +/* Checked-in N-API loader for the seven platform packages this project ships. */ + +const { closeSync, openSync, readSync, readdirSync } = require('fs') +const loadErrors = [] + +const isFileMusl = (file) => file.includes('libc.musl-') || file.includes('ld-musl-') + +const isMuslFromReport = () => { + try { + if (!process.report || typeof process.report.getReport !== 'function') return null + const report = process.report.getReport() + if (report?.header?.glibcVersionRuntime) return false + if (report?.sharedObjects?.some(isFileMusl)) return true + } catch { + // Continue with filesystem and ELF inspection. + } + return null +} + +const isMuslFromFilesystem = () => { + for (const directory of ['/lib', '/usr/lib']) { + try { + if (readdirSync(directory).some(isFileMusl)) return true + } catch { + // A missing or unreadable conventional library directory is inconclusive. + } + } + return null +} + +const readUInt = (buffer, offset, bytes, littleEndian) => { + if (offset < 0 || offset + bytes > buffer.length) return null + if (bytes === 2) return littleEndian ? buffer.readUInt16LE(offset) : buffer.readUInt16BE(offset) + if (bytes === 4) return littleEndian ? buffer.readUInt32LE(offset) : buffer.readUInt32BE(offset) + if (bytes === 8) { + const value = littleEndian ? buffer.readBigUInt64LE(offset) : buffer.readBigUInt64BE(offset) + return value <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(value) : null + } + return null +} + +const isMuslFromElfInterpreter = () => { + let fd + try { + fd = openSync(process.execPath, 'r') + const header = Buffer.alloc(64) + if (readSync(fd, header, 0, header.length, 0) < 52) return null + if (!header.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) return null + + const elfClass = header[4] + const dataEncoding = header[5] + if ((elfClass !== 1 && elfClass !== 2) || (dataEncoding !== 1 && dataEncoding !== 2)) { + return null + } + const littleEndian = dataEncoding === 1 + const is64Bit = elfClass === 2 + const tableOffset = readUInt(header, is64Bit ? 32 : 28, is64Bit ? 8 : 4, littleEndian) + const entrySize = readUInt(header, is64Bit ? 54 : 42, 2, littleEndian) + const entryCount = readUInt(header, is64Bit ? 56 : 44, 2, littleEndian) + if (tableOffset === null || entrySize === null || entryCount === null || + entrySize < (is64Bit ? 56 : 32) || entryCount > 1024) return null + + const programHeader = Buffer.alloc(entrySize) + for (let index = 0; index < entryCount; index += 1) { + const offset = tableOffset + index * entrySize + if (readSync(fd, programHeader, 0, entrySize, offset) !== entrySize) return null + if (readUInt(programHeader, 0, 4, littleEndian) !== 3) continue // PT_INTERP + const interpreterOffset = readUInt(programHeader, is64Bit ? 8 : 4, is64Bit ? 8 : 4, littleEndian) + const interpreterSize = readUInt(programHeader, is64Bit ? 32 : 16, is64Bit ? 8 : 4, littleEndian) + if (interpreterOffset === null || interpreterSize === null || + interpreterSize < 1 || interpreterSize > 4096) return null + const interpreter = Buffer.alloc(interpreterSize) + if (readSync(fd, interpreter, 0, interpreterSize, interpreterOffset) !== interpreterSize) { + return null + } + return isFileMusl(interpreter.toString('utf8')) + } + } catch { + return null + } finally { + if (fd !== undefined) { + try { closeSync(fd) } catch { + // Best-effort detection must never turn close failure into import failure. + } + } + } + return null +} + +const isMusl = () => { + if (process.platform !== 'linux') return false + for (const detector of [isMuslFromReport, isMuslFromFilesystem, isMuslFromElfInterpreter]) { + const result = detector() + if (result !== null) return result + } + // Unknown Linux libc: try the glibc package and let its require fail normally. + return false +} + +const targetSuffix = () => { + if (process.platform === 'win32' && process.arch === 'x64') return 'win32-x64-msvc' + if (process.platform === 'darwin' && process.arch === 'x64') return 'darwin-x64' + if (process.platform === 'darwin' && process.arch === 'arm64') return 'darwin-arm64' + if (process.platform === 'linux' && (process.arch === 'x64' || process.arch === 'arm64')) { + return `linux-${process.arch}-${isMusl() ? 'musl' : 'gnu'}` + } + return null +} + +const tryRequire = (specifier) => { + try { + return require(specifier) + } catch (error) { + loadErrors.push(error) + return null + } +} + +const requireNative = () => { + if (process.env.NAPI_RS_NATIVE_LIBRARY_PATH) { + const explicit = tryRequire(process.env.NAPI_RS_NATIVE_LIBRARY_PATH) + if (explicit) return explicit + } + + const suffix = targetSuffix() + if (!suffix) { + loadErrors.push(new Error(`Unsupported OS or architecture: ${process.platform}-${process.arch}`)) + return null + } + const local = tryRequire(`./fs-safe-native.${suffix}.node`) + if (local) return local + + const packageName = `@openclaw/fs-safe-native-${suffix}` + const binding = tryRequire(packageName) + if (!binding) return null + if (process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + const expectedVersion = require('./package.json').version + const bindingVersion = require(`${packageName}/package.json`).version + if (bindingVersion !== expectedVersion) { + loadErrors.push(new Error( + `Native binding package version mismatch, expected ${expectedVersion} but got ${bindingVersion}. ` + + 'Reinstall dependencies to fix this issue.', + )) + return null + } + } + return binding +} + +const nativeBinding = requireNative() +if (!nativeBinding) { + const error = new Error( + `Cannot find native binding for ${process.platform}-${process.arch}; ` + + 'install the matching optional @openclaw/fs-safe-native platform package.', + ) + error.cause = loadErrors.reduceRight((cause, current) => { + current.cause = cause + return current + }, undefined) + throw error +} + +module.exports = nativeBinding +module.exports.cloneFileExclusive = nativeBinding.cloneFileExclusive +module.exports.copyFileRangeExclusive = nativeBinding.copyFileRangeExclusive +module.exports.createPrivateDirectory = nativeBinding.createPrivateDirectory +module.exports.extractArchiveNative = nativeBinding.extractArchiveNative +module.exports.fstatIdentity = nativeBinding.fstatIdentity +module.exports.inspectArchiveNative = nativeBinding.inspectArchiveNative +module.exports.linkBeneath = nativeBinding.linkBeneath +module.exports.mkdirBeneath = nativeBinding.mkdirBeneath +module.exports.openBeneath = nativeBinding.openBeneath +module.exports.readArchiveEntryNative = nativeBinding.readArchiveEntryNative +module.exports.readOwnerAndDacl = nativeBinding.readOwnerAndDacl +module.exports.renameNoReplace = nativeBinding.renameNoReplace +module.exports.sha256File = nativeBinding.sha256File diff --git a/native/package.json b/native/package.json new file mode 100644 index 0000000..04b0e97 --- /dev/null +++ b/native/package.json @@ -0,0 +1,53 @@ +{ + "name": "@openclaw/fs-safe-native", + "version": "0.5.0-dev.0", + "description": "Native fd-relative filesystem primitives for @openclaw/fs-safe.", + "license": "MIT", + "author": "OpenClaw Team ", + "repository": { + "type": "git", + "url": "git+https://github.com/openclaw/fs-safe.git", + "directory": "native" + }, + "main": "index.js", + "types": "index.d.ts", + "files": [ + "index.js", + "index.d.ts" + ], + "publishConfig": { + "access": "public", + "provenance": true + }, + "engines": { + "node": ">=22" + }, + "napi": { + "binaryName": "fs-safe-native", + "packageName": "@openclaw/fs-safe-native", + "targets": [ + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "x86_64-apple-darwin", + "aarch64-apple-darwin", + "x86_64-pc-windows-msvc" + ] + }, + "scripts": { + "artifacts": "napi artifacts --npm-dir ../npm", + "build": "napi build --platform --release && node ../scripts/harden-native-loader.mjs", + "build:debug": "napi build --platform && node ../scripts/harden-native-loader.mjs", + "prepublishOnly": "napi prepublish -t npm" + }, + "optionalDependencies": { + "@openclaw/fs-safe-native-darwin-arm64": "0.5.0-dev.0", + "@openclaw/fs-safe-native-darwin-x64": "0.5.0-dev.0", + "@openclaw/fs-safe-native-linux-arm64-gnu": "0.5.0-dev.0", + "@openclaw/fs-safe-native-linux-arm64-musl": "0.5.0-dev.0", + "@openclaw/fs-safe-native-linux-x64-gnu": "0.5.0-dev.0", + "@openclaw/fs-safe-native-linux-x64-musl": "0.5.0-dev.0", + "@openclaw/fs-safe-native-win32-x64-msvc": "0.5.0-dev.0" + } +} diff --git a/native/src/archive.rs b/native/src/archive.rs new file mode 100644 index 0000000..2366a98 --- /dev/null +++ b/native/src/archive.rs @@ -0,0 +1,1026 @@ +use std::collections::HashMap; +use std::fs::File; +use std::io::{Read, Seek, SeekFrom}; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; + +use napi::bindgen_prelude::{AbortSignal, AsyncTask, Buffer, Task}; +use napi::{Env, Error, Result, Status}; +use napi_derive::napi; + +use crate::tar_meter::TarMetadataMeter; +use crate::{NativeResult, native_error, platform, validate_relative_path}; + +#[napi(object)] +pub struct NativeArchiveEntry { + pub index: u32, + pub path: String, + pub kind: String, + pub size: f64, + pub mode: u32, +} + +#[napi(object)] +#[derive(Clone)] +pub struct NativeArchivePlanEntry { + pub index: u32, + pub path: String, + pub kind: String, + pub size: f64, + pub mode: u32, +} + +#[derive(Debug)] +pub struct ArchiveEntryData { + index: u32, + path: String, + kind: String, + size: u64, + mode: u32, +} + +#[derive(Clone, Copy)] +enum ArchiveFormat { + Tar, + TarZstd, + TarBzip2, + Zip, +} + +#[derive(Clone, Copy)] +struct InspectLimits { + max_entries: usize, + max_meta_entry_bytes: u64, + max_manifest_bytes: u64, +} + +struct CancellationReader { + inner: R, + cancelled: Arc, +} + +impl Read for CancellationReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + if self.cancelled.load(Ordering::Relaxed) { + return Err(std::io::Error::new( + std::io::ErrorKind::Interrupted, + "archive operation aborted", + )); + } + self.inner.read(buffer) + } +} + +fn parse_format(value: &str) -> NativeResult { + match value { + "tar" => Ok(ArchiveFormat::Tar), + "tar-zstd" => Ok(ArchiveFormat::TarZstd), + "tar-bzip2" => Ok(ArchiveFormat::TarBzip2), + "zip" => Ok(ArchiveFormat::Zip), + _ => Err(native_error( + "EINVAL", + format!("unsupported native archive kind: {value}"), + )), + } +} + +fn io_error(operation: &str, error: impl std::fmt::Display) -> Error { + Error::new(Status::GenericFailure, format!("{operation}: {error}")) +} + +fn open_tar_reader( + path: &str, + format: ArchiveFormat, + cancelled: Arc, + max_meta_entry_bytes: u64, +) -> Result> { + let mut file = File::open(path).map_err(|error| io_error("open archive", error))?; + let decoded: Box = match format { + ArchiveFormat::TarZstd => Box::new(CancellationReader { + inner: zstd::stream::read::Decoder::new(file) + .map_err(|error| io_error("open zstd archive", error))?, + cancelled, + }), + ArchiveFormat::TarBzip2 => Box::new(CancellationReader { + inner: bzip2::read::MultiBzDecoder::new(file), + cancelled, + }), + ArchiveFormat::Tar => { + let mut magic = [0_u8; 2]; + let read = file + .read(&mut magic) + .map_err(|error| io_error("read archive", error))?; + file.seek(SeekFrom::Start(0)) + .map_err(|error| io_error("rewind archive", error))?; + if read == 2 && magic == [0x1f, 0x8b] { + Box::new(CancellationReader { + inner: flate2::read::MultiGzDecoder::new(file), + cancelled, + }) + } else { + Box::new(CancellationReader { + inner: file, + cancelled, + }) + } + } + ArchiveFormat::Zip => { + return Err(Error::new(Status::InvalidArg, "zip is not a tar stream")); + } + }; + Ok(Box::new(TarMetadataMeter::new( + decoded, + max_meta_entry_bytes, + ))) +} + +fn tar_kind(entry_type: tar::EntryType) -> &'static str { + if entry_type.is_dir() { + "directory" + } else if entry_type.is_file() || entry_type.is_contiguous() { + "file" + } else if entry_type.is_gnu_sparse() { + "sparse" + } else if entry_type.is_symlink() { + "symlink" + } else if entry_type.is_hard_link() { + "hardlink" + } else { + "other" + } +} + +fn checked_path(path: std::borrow::Cow<'_, std::path::Path>) -> Result { + path.into_owned() + .into_os_string() + .into_string() + .map_err(|_| Error::new(Status::InvalidArg, "archive entry path is not valid UTF-8")) +} + +fn inspect_tar( + path: &str, + format: ArchiveFormat, + limits: InspectLimits, + cancelled: Arc, +) -> Result> { + let mut archive = tar::Archive::new(open_tar_reader( + path, + format, + Arc::clone(&cancelled), + limits.max_meta_entry_bytes, + )?); + let entries = archive + .entries() + .map_err(|error| io_error("read tar entries", error))?; + let mut result = Vec::new(); + let mut manifest_bytes = 0_u64; + for (index, entry) in entries.enumerate() { + check_cancelled(&cancelled)?; + let entry = entry.map_err(|error| io_error("read tar entry", error))?; + let header = entry.header(); + check_manifest_count(index + 1, limits)?; + let size = entry.size(); + let path = checked_path( + entry + .path() + .map_err(|error| io_error("read tar path", error))?, + )?; + add_manifest_path_bytes(&mut manifest_bytes, &path, limits)?; + result.push(ArchiveEntryData { + index: u32::try_from(index) + .map_err(|_| Error::new(Status::InvalidArg, "too many archive entries"))?, + path, + kind: tar_kind(header.entry_type()).to_owned(), + size, + mode: header.mode().unwrap_or(0), + }); + } + Ok(result) +} + +fn zip_kind(file: &zip::read::ZipFile<'_, File>) -> &'static str { + let mode = file.unix_mode().unwrap_or(0); + if mode & 0o170000 == 0o120000 { + "symlink" + } else if file.is_dir() { + "directory" + } else if file.is_file() { + "file" + } else { + "other" + } +} + +fn inspect_zip( + path: &str, + limits: InspectLimits, + cancelled: &AtomicBool, +) -> Result> { + zip_entry_count(path, limits.max_entries)?; + let file = File::open(path).map_err(|error| io_error("open zip archive", error))?; + let mut archive = + zip::ZipArchive::new(file).map_err(|error| io_error("read zip archive", error))?; + let mut result = Vec::with_capacity(archive.len()); + let mut manifest_bytes = 0_u64; + for index in 0..archive.len() { + check_cancelled(cancelled)?; + let file = archive + .by_index(index) + .map_err(|error| io_error("read zip entry", error))?; + add_manifest_path_bytes(&mut manifest_bytes, file.name(), limits)?; + result.push(ArchiveEntryData { + index: u32::try_from(index) + .map_err(|_| Error::new(Status::InvalidArg, "too many archive entries"))?, + path: file.name().to_owned(), + kind: zip_kind(&file).to_owned(), + size: file.size(), + mode: file.unix_mode().unwrap_or(0), + }); + } + Ok(result) +} + +fn inspect( + path: &str, + format: ArchiveFormat, + limits: InspectLimits, + cancelled: Arc, +) -> Result> { + match format { + ArchiveFormat::Zip => inspect_zip(path, limits, &cancelled), + _ => inspect_tar(path, format, limits, cancelled), + } +} + +fn limit_error(code: &'static str) -> Error { + Error::new(Status::GenericFailure, code) +} + +fn check_manifest_count(count: usize, limits: InspectLimits) -> Result<()> { + if count > limits.max_entries { + return Err(limit_error("archive-entry-count-exceeds-limit")); + } + Ok(()) +} + +fn add_manifest_path_bytes(total: &mut u64, path: &str, limits: InspectLimits) -> Result<()> { + *total = total + .checked_add(path.len() as u64) + .ok_or_else(|| limit_error("archive-manifest-size-exceeds-limit"))?; + if *total > limits.max_manifest_bytes { + return Err(limit_error("archive-manifest-size-exceeds-limit")); + } + Ok(()) +} + +fn zip_entry_count(path: &str, max_entries: usize) -> Result { + let mut file = File::open(path).map_err(|error| io_error("open zip archive", error))?; + let length = file + .metadata() + .map_err(|error| io_error("stat zip archive", error))? + .len(); + let tail_size = length.min(65_557) as usize; + let mut tail = vec![0_u8; tail_size]; + file.seek(SeekFrom::End(-(tail_size as i64))) + .and_then(|_| file.read_exact(&mut tail)) + .map_err(|error| io_error("read zip directory", error))?; + for eocd in (0..tail.len().saturating_sub(21)).rev() { + if tail[eocd..eocd + 4] != [0x50, 0x4b, 0x05, 0x06] { + continue; + } + let comment_length = u16::from_le_bytes([tail[eocd + 20], tail[eocd + 21]]) as usize; + if eocd + 22 + comment_length != tail.len() { + continue; + } + let mut count = u16::from_le_bytes([tail[eocd + 10], tail[eocd + 11]]) as u64; + let mut directory_size = + u32::from_le_bytes(tail[eocd + 12..eocd + 16].try_into().unwrap()) as u64; + let mut directory_offset = + u32::from_le_bytes(tail[eocd + 16..eocd + 20].try_into().unwrap()) as u64; + let absolute_eocd = length - tail_size as u64 + eocd as u64; + let mut expected_directory_end = absolute_eocd; + if count == u16::MAX as u64 + || directory_size == u32::MAX as u64 + || directory_offset == u32::MAX as u64 + { + if eocd < 20 || tail[eocd - 20..eocd - 16] != [0x50, 0x4b, 0x06, 0x07] { + continue; + } + let record_offset = u64::from_le_bytes(tail[eocd - 12..eocd - 4].try_into().unwrap()); + let mut record = [0_u8; 56]; + if file + .seek(SeekFrom::Start(record_offset)) + .and_then(|_| file.read_exact(&mut record)) + .is_err() + || record[..4] != [0x50, 0x4b, 0x06, 0x06] + { + continue; + } + count = u64::from_le_bytes(record[32..40].try_into().unwrap()); + directory_size = u64::from_le_bytes(record[40..48].try_into().unwrap()); + directory_offset = u64::from_le_bytes(record[48..56].try_into().unwrap()); + expected_directory_end = record_offset; + } + if count > max_entries as u64 { + return Err(limit_error("archive-entry-count-exceeds-limit")); + } + if directory_offset.checked_add(directory_size) != Some(expected_directory_end) { + continue; + } + let parsed = count_central_directory_entries( + &mut file, + directory_offset, + directory_size, + max_entries, + )?; + if parsed == count { + return Ok(parsed); + } + } + Err(Error::new( + Status::InvalidArg, + "valid zip end-of-central-directory record missing", + )) +} + +fn count_central_directory_entries( + file: &mut File, + offset: u64, + size: u64, + max_entries: usize, +) -> Result { + file.seek(SeekFrom::Start(offset)) + .map_err(|error| io_error("seek zip directory", error))?; + let mut consumed = 0_u64; + let mut count = 0_u64; + while consumed < size { + let mut header = [0_u8; 46]; + file.read_exact(&mut header) + .map_err(|error| io_error("read zip directory entry", error))?; + if header[..4] != [0x50, 0x4b, 0x01, 0x02] { + return Err(Error::new( + Status::InvalidArg, + "invalid zip central directory entry", + )); + } + let variable = u16::from_le_bytes([header[28], header[29]]) as u64 + + u16::from_le_bytes([header[30], header[31]]) as u64 + + u16::from_le_bytes([header[32], header[33]]) as u64; + consumed = consumed + .checked_add(46 + variable) + .ok_or_else(|| Error::new(Status::InvalidArg, "zip directory size overflow"))?; + if consumed > size { + return Err(Error::new( + Status::InvalidArg, + "truncated zip central directory", + )); + } + file.seek(SeekFrom::Current(variable as i64)) + .map_err(|error| io_error("skip zip directory entry", error))?; + count += 1; + if count > max_entries as u64 { + return Err(limit_error("archive-entry-count-exceeds-limit")); + } + } + Ok(count) +} + +fn check_cancelled(cancelled: &AtomicBool) -> Result<()> { + if cancelled.load(Ordering::Relaxed) { + Err(Error::new(Status::Cancelled, "archive operation aborted")) + } else { + Ok(()) + } +} + +fn cancellation(signal: &AbortSignal) -> Arc { + let cancelled = Arc::new(AtomicBool::new(false)); + let callback = Arc::clone(&cancelled); + signal.on_abort(move || callback.store(true, Ordering::Relaxed)); + cancelled +} + +pub struct InspectTask { + path: String, + format: ArchiveFormat, + limits: InspectLimits, + cancelled: Arc, +} + +impl Task for InspectTask { + type Output = Vec; + type JsValue = Vec; + + fn compute(&mut self) -> Result { + inspect( + &self.path, + self.format, + self.limits, + Arc::clone(&self.cancelled), + ) + } + + fn resolve(&mut self, _env: Env, output: Self::Output) -> Result { + Ok(output + .into_iter() + .map(|entry| NativeArchiveEntry { + index: entry.index, + path: entry.path, + kind: entry.kind, + size: entry.size as f64, + mode: entry.mode, + }) + .collect()) + } +} + +#[napi(js_name = "inspectArchiveNative")] +pub fn inspect_archive_native( + path: String, + kind: String, + max_entries: u32, + max_meta_entry_bytes: f64, + max_manifest_bytes: f64, + signal: AbortSignal, +) -> Result> { + let format = + parse_format(&kind).map_err(|error| Error::new(Status::InvalidArg, error.reason))?; + let limits = InspectLimits { + max_entries: max_entries as usize, + max_meta_entry_bytes: checked_limit(max_meta_entry_bytes, "maxMetaEntryBytes")?, + max_manifest_bytes: checked_limit(max_manifest_bytes, "maxManifestBytes")?, + }; + let cancelled = cancellation(&signal); + Ok(AsyncTask::with_signal( + InspectTask { + path, + format, + limits, + cancelled, + }, + signal, + )) +} + +fn checked_limit(value: f64, label: &str) -> Result { + if !value.is_finite() || value < 0.0 || value > u64::MAX as f64 { + return Err(Error::new( + Status::InvalidArg, + format!("{label} is out of range"), + )); + } + Ok(value as u64) +} + +fn plan_map(plan: Vec) -> Result> { + let mut entries = HashMap::with_capacity(plan.len()); + for entry in plan { + validate_relative_path(&entry.path, false) + .map_err(|error| Error::new(Status::InvalidArg, error.reason))?; + if entry.kind != "file" && entry.kind != "directory" { + return Err(Error::new( + Status::InvalidArg, + "native archive plan only accepts files and directories", + )); + } + let index = entry.index as usize; + if entries.insert(index, entry).is_some() { + return Err(Error::new( + Status::InvalidArg, + "native archive plan contains a duplicate index", + )); + } + } + Ok(entries) +} + +fn ensure_parent(root_fd: i32, path: &str) -> Result<()> { + if let Some((parent, _)) = path.rsplit_once('/') { + platform::mkdir_beneath(root_fd, parent, 0o700) + .map_err(|error| Error::new(Status::GenericFailure, error.reason))?; + } + Ok(()) +} + +fn extract_tar( + path: &str, + format: ArchiveFormat, + root_fd: i32, + mut plan: HashMap, + cancelled: Arc, + max_meta_entry_bytes: u64, +) -> Result<()> { + let mut archive = tar::Archive::new(open_tar_reader( + path, + format, + Arc::clone(&cancelled), + max_meta_entry_bytes, + )?); + let entries = archive + .entries() + .map_err(|error| io_error("read tar entries", error))?; + let mut directories = Vec::new(); + for (index, entry) in entries.enumerate() { + check_cancelled(&cancelled)?; + let Some(item) = plan.remove(&index) else { + continue; + }; + let mut entry = entry.map_err(|error| io_error("read tar entry", error))?; + let actual_kind = tar_kind(entry.header().entry_type()); + if actual_kind != item.kind || entry.size() as f64 != item.size { + return Err(Error::new( + Status::InvalidArg, + "archive entry changed after policy evaluation", + )); + } + if item.kind == "directory" { + platform::mkdir_beneath(root_fd, &item.path, 0o700) + .map_err(|error| Error::new(Status::GenericFailure, error.reason))?; + directories.push((item.path, item.mode)); + } else { + ensure_parent(root_fd, &item.path)?; + let size = entry.size(); + let mut reader = CancellationReader { + inner: &mut entry, + cancelled: Arc::clone(&cancelled), + }; + platform::write_archive_file(root_fd, &item.path, &mut reader, size, item.mode) + .map_err(|error| Error::new(Status::GenericFailure, error.reason))?; + } + } + finish_directories(root_fd, directories)?; + if !plan.is_empty() { + return Err(Error::new( + Status::InvalidArg, + "archive entries disappeared after policy evaluation", + )); + } + Ok(()) +} + +fn extract_zip( + path: &str, + root_fd: i32, + mut plan: HashMap, + cancelled: Arc, +) -> Result<()> { + let file = File::open(path).map_err(|error| io_error("open zip archive", error))?; + let mut archive = + zip::ZipArchive::new(file).map_err(|error| io_error("read zip archive", error))?; + let mut directories = Vec::new(); + for index in 0..archive.len() { + check_cancelled(&cancelled)?; + let Some(item) = plan.remove(&index) else { + continue; + }; + let mut file = archive + .by_index(index) + .map_err(|error| io_error("read zip entry", error))?; + let actual_kind = zip_kind(&file); + if actual_kind != item.kind || file.size() as f64 != item.size { + return Err(Error::new( + Status::InvalidArg, + "archive entry changed after policy evaluation", + )); + } + if item.kind == "directory" { + platform::mkdir_beneath(root_fd, &item.path, 0o700) + .map_err(|error| Error::new(Status::GenericFailure, error.reason))?; + directories.push((item.path, item.mode)); + } else { + ensure_parent(root_fd, &item.path)?; + let size = file.size(); + let mut reader = CancellationReader { + inner: &mut file, + cancelled: Arc::clone(&cancelled), + }; + platform::write_archive_file(root_fd, &item.path, &mut reader, size, item.mode) + .map_err(|error| Error::new(Status::GenericFailure, error.reason))?; + } + } + finish_directories(root_fd, directories)?; + if !plan.is_empty() { + return Err(Error::new( + Status::InvalidArg, + "archive entries disappeared after policy evaluation", + )); + } + Ok(()) +} + +fn finish_directories(root_fd: i32, mut directories: Vec<(String, u32)>) -> Result<()> { + directories.sort_by_key(|(path, _)| std::cmp::Reverse(path.matches('/').count())); + for (path, mode) in directories { + platform::chmod_beneath(root_fd, &path, mode) + .map_err(|error| Error::new(Status::GenericFailure, error.reason))?; + } + Ok(()) +} + +pub struct ExtractTask { + path: String, + format: ArchiveFormat, + root_fd: i32, + plan: Option>, + cancelled: Arc, + max_meta_entry_bytes: u64, +} + +impl Task for ExtractTask { + type Output = (); + type JsValue = (); + + fn compute(&mut self) -> Result { + let plan = plan_map(self.plan.take().unwrap_or_default())?; + match self.format { + ArchiveFormat::Zip => { + extract_zip(&self.path, self.root_fd, plan, Arc::clone(&self.cancelled)) + } + _ => extract_tar( + &self.path, + self.format, + self.root_fd, + plan, + Arc::clone(&self.cancelled), + self.max_meta_entry_bytes, + ), + } + } + + fn resolve(&mut self, _env: Env, output: Self::Output) -> Result { + Ok(output) + } +} + +#[napi(js_name = "extractArchiveNative")] +pub fn extract_archive_native( + path: String, + kind: String, + root_fd: i32, + plan: Vec, + max_meta_entry_bytes: f64, + signal: AbortSignal, +) -> Result> { + let format = + parse_format(&kind).map_err(|error| Error::new(Status::InvalidArg, error.reason))?; + let cancelled = cancellation(&signal); + let max_meta_entry_bytes = checked_limit(max_meta_entry_bytes, "maxMetaEntryBytes")?; + Ok(AsyncTask::with_signal( + ExtractTask { + path, + format, + root_fd, + plan: Some(plan), + cancelled, + max_meta_entry_bytes, + }, + signal, + )) +} + +fn read_tar_entry( + path: &str, + format: ArchiveFormat, + requested: &str, + max_bytes: u64, + cancelled: Arc, + max_meta_entry_bytes: u64, + max_entries: usize, +) -> Result> { + let mut archive = tar::Archive::new(open_tar_reader( + path, + format, + Arc::clone(&cancelled), + max_meta_entry_bytes, + )?); + for (index, entry) in archive + .entries() + .map_err(|error| io_error("read tar entries", error))? + .enumerate() + { + if index >= max_entries { + return Err(limit_error("archive-entry-count-exceeds-limit")); + } + check_cancelled(&cancelled)?; + let mut entry = entry.map_err(|error| io_error("read tar entry", error))?; + if checked_path( + entry + .path() + .map_err(|error| io_error("read tar path", error))?, + )? != requested + { + continue; + } + if tar_kind(entry.header().entry_type()) != "file" { + if entry.header().entry_type().is_gnu_sparse() { + return Err(Error::new( + Status::InvalidArg, + "archive-header-invalid: GNU sparse entries are not supported", + )); + } + return Err(Error::new( + Status::InvalidArg, + format!("archive entry is not a file: {requested}"), + )); + } + return read_bounded(&mut entry, max_bytes, cancelled); + } + Err(Error::new( + Status::InvalidArg, + format!("archive entry not found: {requested}"), + )) +} + +fn read_zip_entry( + path: &str, + requested: &str, + max_bytes: u64, + max_entries: usize, + cancelled: Arc, +) -> Result> { + zip_entry_count(path, max_entries)?; + let file = File::open(path).map_err(|error| io_error("open zip archive", error))?; + let mut archive = + zip::ZipArchive::new(file).map_err(|error| io_error("read zip archive", error))?; + let mut entry = archive.by_name(requested).map_err(|_| { + Error::new( + Status::InvalidArg, + format!("archive entry not found: {requested}"), + ) + })?; + if zip_kind(&entry) != "file" { + return Err(Error::new( + Status::InvalidArg, + format!("archive entry is not a file: {requested}"), + )); + } + read_bounded(&mut entry, max_bytes, cancelled) +} + +fn read_bounded( + reader: &mut impl Read, + max_bytes: u64, + cancelled: Arc, +) -> Result> { + let capacity = usize::try_from(max_bytes.min(1024 * 1024)).unwrap_or(0); + let mut output = Vec::with_capacity(capacity); + CancellationReader { + inner: reader, + cancelled, + } + .take(max_bytes.saturating_add(1)) + .read_to_end(&mut output) + .map_err(|error| io_error("read archive entry", error))?; + if output.len() as u64 > max_bytes { + return Err(Error::new( + Status::GenericFailure, + "archive-entry-extracted-size-exceeds-limit", + )); + } + Ok(output) +} + +pub struct ReadEntryTask { + path: String, + format: ArchiveFormat, + requested: String, + max_bytes: u64, + max_entries: usize, + cancelled: Arc, + max_meta_entry_bytes: u64, +} + +impl Task for ReadEntryTask { + type Output = Vec; + type JsValue = Buffer; + fn compute(&mut self) -> Result { + check_cancelled(&self.cancelled)?; + match self.format { + ArchiveFormat::Zip => read_zip_entry( + &self.path, + &self.requested, + self.max_bytes, + self.max_entries, + Arc::clone(&self.cancelled), + ), + _ => read_tar_entry( + &self.path, + self.format, + &self.requested, + self.max_bytes, + Arc::clone(&self.cancelled), + self.max_meta_entry_bytes, + self.max_entries, + ), + } + } + fn resolve(&mut self, _env: Env, output: Self::Output) -> Result { + Ok(output.into()) + } +} + +#[napi(js_name = "readArchiveEntryNative")] +pub fn read_archive_entry_native( + path: String, + kind: String, + requested: String, + max_bytes: f64, + max_entries: u32, + max_meta_entry_bytes: f64, + signal: AbortSignal, +) -> Result> { + if !max_bytes.is_finite() || max_bytes < 0.0 || max_bytes > u64::MAX as f64 { + return Err(Error::new( + Status::InvalidArg, + "maxBytes must be a non-negative finite number", + )); + } + let format = + parse_format(&kind).map_err(|error| Error::new(Status::InvalidArg, error.reason))?; + let cancelled = cancellation(&signal); + let max_meta_entry_bytes = checked_limit(max_meta_entry_bytes, "maxMetaEntryBytes")?; + Ok(AsyncTask::with_signal( + ReadEntryTask { + path, + format, + requested, + max_bytes: max_bytes as u64, + max_entries: max_entries as usize, + cancelled, + max_meta_entry_bytes, + }, + signal, + )) +} + +#[cfg(test)] +mod tests { + use std::io::{Cursor, Write}; + use std::time::{SystemTime, UNIX_EPOCH}; + + use super::*; + + fn fixture_tar() -> Vec { + let mut bytes = Vec::new(); + { + let mut builder = tar::Builder::new(&mut bytes); + for (name, body) in [ + ("one.txt", b"one".as_slice()), + ("two.txt", b"two".as_slice()), + ] { + let mut header = tar::Header::new_ustar(); + header.set_size(body.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder.append_data(&mut header, name, body).unwrap(); + } + builder.finish().unwrap(); + } + bytes + } + + fn temp_path(suffix: &str) -> std::path::PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + std::env::temp_dir().join(format!( + "fs-safe-archive-{}-{nonce}.{suffix}", + std::process::id() + )) + } + + fn limits(max_entries: usize) -> InspectLimits { + InspectLimits { + max_entries, + max_meta_entry_bytes: 1024 * 1024, + max_manifest_bytes: 16 * 1024 * 1024, + } + } + + fn gzip(part: &[u8]) -> Vec { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(part).unwrap(); + encoder.finish().unwrap() + } + + fn bzip(part: &[u8]) -> Vec { + let mut encoder = bzip2::write::BzEncoder::new(Vec::new(), bzip2::Compression::fast()); + encoder.write_all(part).unwrap(); + encoder.finish().unwrap() + } + + #[test] + fn manifest_is_bounded_before_result_allocation() { + let path = temp_path("tar"); + std::fs::write(&path, fixture_tar()).unwrap(); + let error = inspect_tar( + path.to_str().unwrap(), + ArchiveFormat::Tar, + limits(1), + Arc::new(AtomicBool::new(false)), + ) + .unwrap_err(); + assert!(error.reason.contains("archive-entry-count-exceeds-limit")); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn manifest_paths_and_single_entry_reads_are_bounded() { + let path = temp_path("tar"); + std::fs::write(&path, fixture_tar()).unwrap(); + let mut bounded = limits(10); + bounded.max_manifest_bytes = 3; + let error = inspect_tar( + path.to_str().unwrap(), + ArchiveFormat::Tar, + bounded, + Arc::new(AtomicBool::new(false)), + ) + .unwrap_err(); + assert!(error.reason.contains("archive-manifest-size-exceeds-limit")); + + let error = read_tar_entry( + path.to_str().unwrap(), + ArchiveFormat::Tar, + "missing", + 1024, + Arc::new(AtomicBool::new(false)), + 1024, + 1, + ) + .unwrap_err(); + assert!(error.reason.contains("archive-entry-count-exceeds-limit")); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn cancellation_reader_checks_every_read() { + let cancelled = Arc::new(AtomicBool::new(true)); + let mut reader = CancellationReader { + inner: Cursor::new(vec![1_u8; 16]), + cancelled, + }; + assert_eq!( + reader.read(&mut [0_u8; 8]).unwrap_err().kind(), + std::io::ErrorKind::Interrupted + ); + } + + #[test] + fn gzip_and_bzip2_decode_concatenated_members() { + type FormatFixture = (ArchiveFormat, &'static str, fn(&[u8]) -> Vec); + let tar = fixture_tar(); + let formats: [FormatFixture; 2] = [ + (ArchiveFormat::Tar, "tar.gz", gzip), + (ArchiveFormat::TarBzip2, "tar.bz2", bzip), + ]; + for (format, suffix, encode) in formats { + let midpoint = tar.len() / 2; + let bytes = [encode(&tar[..midpoint]), encode(&tar[midpoint..])].concat(); + let path = temp_path(suffix); + std::fs::write(&path, bytes).unwrap(); + let entries = inspect_tar( + path.to_str().unwrap(), + format, + limits(10), + Arc::new(AtomicBool::new(false)), + ) + .unwrap(); + assert_eq!(entries.len(), 2); + std::fs::remove_file(path).unwrap(); + } + } + + #[test] + fn zip_preflight_rejects_fake_eocd_comment_candidates() { + let cursor = Cursor::new(Vec::new()); + let mut writer = zip::ZipWriter::new(cursor); + for name in ["one", "two"] { + writer + .start_file(name, zip::write::SimpleFileOptions::default()) + .unwrap(); + writer.write_all(b"x").unwrap(); + } + let mut bytes = writer.finish().unwrap().into_inner(); + let real_eocd = bytes + .windows(4) + .rposition(|window| window == [0x50, 0x4b, 0x05, 0x06]) + .unwrap(); + bytes[real_eocd + 20..real_eocd + 22].copy_from_slice(&22_u16.to_le_bytes()); + let mut fake = [0_u8; 22]; + fake[..4].copy_from_slice(&[0x50, 0x4b, 0x05, 0x06]); + fake[8..10].copy_from_slice(&1_u16.to_le_bytes()); + fake[10..12].copy_from_slice(&1_u16.to_le_bytes()); + bytes.extend_from_slice(&fake); + let path = temp_path("zip"); + std::fs::write(&path, bytes).unwrap(); + + let error = zip_entry_count(path.to_str().unwrap(), 1).unwrap_err(); + assert!(error.reason.contains("archive-entry-count-exceeds-limit")); + assert_eq!(zip_entry_count(path.to_str().unwrap(), 2).unwrap(), 2); + std::fs::remove_file(path).unwrap(); + } +} diff --git a/native/src/fast_file.rs b/native/src/fast_file.rs new file mode 100644 index 0000000..aabf3ce --- /dev/null +++ b/native/src/fast_file.rs @@ -0,0 +1,130 @@ +use napi::bindgen_prelude::{AsyncTask, Task}; +use napi::{Env, Error, Result, Status}; +use napi_derive::napi; +use sha2::{Digest, Sha256}; +use std::fmt::Write as _; + +use crate::{into_napi, platform}; + +#[napi(object)] +pub struct FileHash { + pub bytes: f64, + pub digest: String, +} + +#[napi(object)] +pub struct NativeCopyResult { + pub fd: i32, + pub bytes: f64, + pub error_code: Option, + pub error_message: Option, +} + +#[napi(js_name = "cloneFileExclusive")] +pub fn clone_file_exclusive( + env: Env, + source_fd: i32, + target_root_fd: i32, + target_rel_path: String, +) -> Result { + into_napi( + env, + crate::validate_relative_path(&target_rel_path, false).and_then(|()| { + platform::clone_file_exclusive(source_fd, target_root_fd, &target_rel_path) + }), + ) +} + +pub struct CopyFileRangeTask { + source_fd: i32, + target_root_fd: i32, + target_rel_path: String, +} + +impl Task for CopyFileRangeTask { + type Output = crate::NativeResult<(i32, u64)>; + type JsValue = NativeCopyResult; + + fn compute(&mut self) -> Result { + Ok(platform::copy_file_range_exclusive( + self.source_fd, + self.target_root_fd, + &self.target_rel_path, + )) + } + + fn resolve(&mut self, _env: Env, output: Self::Output) -> Result { + Ok(match output { + Ok((fd, bytes)) => NativeCopyResult { + fd, + bytes: bytes as f64, + error_code: None, + error_message: None, + }, + Err(error) => NativeCopyResult { + fd: -1, + bytes: 0.0, + error_code: Some(error.status), + error_message: Some(error.reason), + }, + }) + } +} + +#[napi(js_name = "copyFileRangeExclusive")] +pub fn copy_file_range_exclusive( + source_fd: i32, + target_root_fd: i32, + target_rel_path: String, +) -> Result> { + crate::validate_relative_path(&target_rel_path, false) + .map_err(|error| Error::new(Status::InvalidArg, error.reason))?; + Ok(AsyncTask::new(CopyFileRangeTask { + source_fd, + target_root_fd, + target_rel_path, + })) +} + +pub struct HashTask { + fd: i32, +} + +impl Task for HashTask { + type Output = (u64, String); + type JsValue = FileHash; + + fn compute(&mut self) -> Result { + let reader = platform::open_independent_reader(self.fd) + .map_err(|error| Error::new(Status::GenericFailure, error.reason))?; + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + let mut bytes = 0_u64; + loop { + let read = platform::read_at(&reader, &mut buffer, bytes) + .map_err(|error| Error::new(Status::GenericFailure, error.reason))?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + bytes += read as u64; + } + let mut digest = String::with_capacity(64); + for byte in hasher.finalize() { + write!(&mut digest, "{byte:02x}").unwrap(); + } + Ok((bytes, digest)) + } + + fn resolve(&mut self, _env: Env, (bytes, digest): Self::Output) -> Result { + Ok(FileHash { + bytes: bytes as f64, + digest, + }) + } +} + +#[napi(js_name = "sha256File")] +pub fn sha256_file(fd: i32) -> AsyncTask { + AsyncTask::new(HashTask { fd }) +} diff --git a/native/src/lib.rs b/native/src/lib.rs new file mode 100644 index 0000000..d1607f5 --- /dev/null +++ b/native/src/lib.rs @@ -0,0 +1,154 @@ +#![deny(unsafe_op_in_unsafe_fn)] + +use napi::bindgen_prelude::*; +use napi_derive::napi; + +mod archive; +mod fast_file; +mod tar_meter; +#[cfg(unix)] +mod unix; +#[cfg(windows)] +mod windows; +mod windows_security; + +#[napi(object)] +pub struct FileIdentity { + pub dev: f64, + pub ino: f64, + pub mode: u32, + pub nlink: f64, + pub size: f64, + pub is_file: bool, + pub is_directory: bool, + pub is_symbolic_link: bool, +} + +pub(crate) type NativeResult = std::result::Result>; + +pub(crate) fn native_error(code: impl Into, message: impl Into) -> Error { + Error::new(code.into(), message.into()) +} + +fn invalid_path(message: impl Into) -> Error { + native_error("EINVAL", message) +} + +fn validate_relative_path(path: &str, allow_root: bool) -> NativeResult<()> { + if path.as_bytes().contains(&0) { + return Err(invalid_path("relative path contains a NUL byte")); + } + if path.is_empty() || path == "." { + return if allow_root { + Ok(()) + } else { + Err(invalid_path("operation requires a non-root path")) + }; + } + if path.starts_with('/') || path.starts_with('\\') { + return Err(invalid_path( + "path must be relative to the supplied root descriptor", + )); + } + if path.split(['/', '\\']).any(|segment| segment == "..") { + return Err(invalid_path("relative path must not contain '..'")); + } + Ok(()) +} + +fn into_napi(env: Env, result: NativeResult) -> Result { + match result { + Ok(value) => Ok(value), + Err(error) => { + let reason = error.reason; + env.throw_error(&reason, Some(error.status.as_ref()))?; + Err(Error::new(Status::PendingException, reason)) + } + } +} + +#[napi(js_name = "openBeneath")] +pub fn open_beneath(env: Env, root_fd: i32, rel_path: String, flags: i32) -> Result { + into_napi( + env, + validate_relative_path(&rel_path, true) + .and_then(|()| platform::open_beneath(root_fd, &rel_path, flags)), + ) +} + +#[napi(js_name = "mkdirBeneath")] +pub fn mkdir_beneath(env: Env, root_fd: i32, rel_path: String, mode: u32) -> Result<()> { + into_napi( + env, + validate_relative_path(&rel_path, true) + .and_then(|()| platform::mkdir_beneath(root_fd, &rel_path, mode)), + ) +} + +#[napi(js_name = "linkBeneath")] +pub fn link_beneath( + env: Env, + source_root_fd: i32, + source_rel_path: String, + target_root_fd: i32, + target_rel_path: String, +) -> Result<()> { + into_napi( + env, + validate_relative_path(&source_rel_path, false) + .and_then(|()| validate_relative_path(&target_rel_path, false)) + .and_then(|()| { + platform::link_beneath( + source_root_fd, + &source_rel_path, + target_root_fd, + &target_rel_path, + ) + }), + ) +} + +#[napi(js_name = "renameNoReplace")] +pub fn rename_no_replace( + env: Env, + source_root_fd: i32, + source_rel_path: String, + target_root_fd: i32, + target_rel_path: String, +) -> Result<()> { + into_napi( + env, + validate_relative_path(&source_rel_path, false) + .and_then(|()| validate_relative_path(&target_rel_path, false)) + .and_then(|()| { + platform::rename_no_replace( + source_root_fd, + &source_rel_path, + target_root_fd, + &target_rel_path, + ) + }), + ) +} + +#[napi(js_name = "fstatIdentity")] +pub fn fstat_identity(env: Env, fd: i32) -> Result { + into_napi(env, platform::fstat_identity(fd)) +} + +pub use archive::{ + NativeArchiveEntry, NativeArchivePlanEntry, extract_archive_native, inspect_archive_native, + read_archive_entry_native, +}; +pub use fast_file::{ + FileHash, NativeCopyResult, clone_file_exclusive, copy_file_range_exclusive, sha256_file, +}; +pub use windows_security::{ + WindowsAccessControlEntry, WindowsAceFlags, WindowsSecurityFacts, create_private_directory, + read_owner_and_dacl, +}; + +#[cfg(unix)] +use unix as platform; +#[cfg(windows)] +use windows as platform; diff --git a/native/src/tar_meter.rs b/native/src/tar_meter.rs new file mode 100644 index 0000000..6abb4cc --- /dev/null +++ b/native/src/tar_meter.rs @@ -0,0 +1,266 @@ +use std::io::{self, Read}; + +pub const INVALID_HEADER: &str = "archive-header-invalid"; +pub const META_LIMIT: &str = "archive-meta-entry-size-exceeds-limit"; + +enum MeterState { + Header, + Data { + remaining: u64, + }, + SparseHeader { + data_remaining: u64, + meta_bytes: u64, + }, +} + +pub struct TarMetadataMeter { + inner: R, + max_meta_entry_bytes: u64, + state: MeterState, + block: [u8; 512], + block_len: usize, +} + +impl TarMetadataMeter { + pub fn new(inner: R, max_meta_entry_bytes: u64) -> Self { + Self { + inner, + max_meta_entry_bytes, + state: MeterState::Header, + block: [0; 512], + block_len: 0, + } + } + + fn invalid(message: &str) -> io::Error { + io::Error::new( + io::ErrorKind::InvalidData, + format!("{INVALID_HEADER}: {message}"), + ) + } + + fn meta_limit() -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, META_LIMIT) + } + + fn padded_size(size: u64) -> io::Result { + size.checked_add(511) + .map(|value| value / 512 * 512) + .ok_or_else(|| Self::invalid("entry size overflows TAR padding")) + } + + fn parse_size(field: &[u8; 12]) -> io::Result { + if field[0] & 0x80 != 0 { + let mut value = 0_u64; + for byte in &field[4..] { + value = value + .checked_shl(8) + .ok_or_else(|| Self::invalid("base-256 size overflow"))?; + value |= *byte as u64; + } + return Ok(value); + } + let end = field + .iter() + .position(|byte| *byte == 0) + .unwrap_or(field.len()); + let text = std::str::from_utf8(&field[..end]) + .map_err(|_| Self::invalid("size is not ASCII octal"))? + .trim(); + if text.is_empty() || !text.bytes().all(|byte| (b'0'..=b'7').contains(&byte)) { + return Err(Self::invalid("size is not valid octal")); + } + u64::from_str_radix(text, 8).map_err(|_| Self::invalid("octal size overflow")) + } + + fn finish_header(&mut self) -> io::Result<()> { + if self.block.iter().all(|byte| *byte == 0) { + self.state = MeterState::Header; + self.block_len = 0; + return Ok(()); + } + let size = Self::parse_size(self.block[124..136].try_into().unwrap())?; + let padded = Self::padded_size(size)?; + let entry_type = self.block[156]; + if matches!(entry_type, b'x' | b'g' | b'L' | b'K' | b'X') + && size > self.max_meta_entry_bytes + { + return Err(Self::meta_limit()); + } + if matches!(entry_type, b'x' | b'g' | b'X') { + return Err(Self::invalid( + "PAX metadata is unmeterable without interpreting content", + )); + } + self.state = if entry_type == b'S' { + match self.block[482] { + 0 => return Err(Self::invalid("GNU sparse entries are not supported")), + 1 => MeterState::SparseHeader { + data_remaining: padded, + meta_bytes: 0, + }, + _ => return Err(Self::invalid("GNU sparse extension flag is not 0 or 1")), + } + } else { + MeterState::Data { remaining: padded } + }; + self.block_len = 0; + if matches!(self.state, MeterState::Data { remaining: 0 }) { + self.state = MeterState::Header; + } + Ok(()) + } + + fn finish_sparse_header(&mut self, data_remaining: u64, meta_bytes: u64) -> io::Result<()> { + let metered = meta_bytes.checked_add(512).ok_or_else(Self::meta_limit)?; + if metered > self.max_meta_entry_bytes { + return Err(Self::meta_limit()); + } + self.state = match self.block[504] { + 0 => return Err(Self::invalid("GNU sparse entries are not supported")), + 1 => MeterState::SparseHeader { + data_remaining, + meta_bytes: metered, + }, + _ => return Err(Self::invalid("GNU sparse extension flag is not 0 or 1")), + }; + self.block_len = 0; + Ok(()) + } + + fn meter(&mut self, bytes: &[u8]) -> io::Result<()> { + let mut offset = 0; + while offset < bytes.len() { + match self.state { + MeterState::Header => { + let take = (512 - self.block_len).min(bytes.len() - offset); + self.block[self.block_len..self.block_len + take] + .copy_from_slice(&bytes[offset..offset + take]); + self.block_len += take; + offset += take; + if self.block_len == 512 { + self.finish_header()?; + } + } + MeterState::Data { remaining } => { + let take = remaining.min((bytes.len() - offset) as u64); + offset += take as usize; + let remaining = remaining - take; + self.state = if remaining == 0 { + MeterState::Header + } else { + MeterState::Data { remaining } + }; + } + MeterState::SparseHeader { + data_remaining, + meta_bytes, + } => { + let take = (512 - self.block_len).min(bytes.len() - offset); + self.block[self.block_len..self.block_len + take] + .copy_from_slice(&bytes[offset..offset + take]); + self.block_len += take; + offset += take; + if self.block_len == 512 { + self.finish_sparse_header(data_remaining, meta_bytes)?; + } + } + } + } + Ok(()) + } + + fn check_eof(&self) -> io::Result<()> { + match self.state { + MeterState::Header if self.block_len == 0 => Ok(()), + MeterState::Header => Err(Self::invalid("truncated TAR header")), + MeterState::Data { .. } => Err(Self::invalid("truncated TAR entry data")), + MeterState::SparseHeader { .. } => Err(Self::invalid("truncated GNU sparse header")), + } + } +} + +impl Read for TarMetadataMeter { + fn read(&mut self, output: &mut [u8]) -> io::Result { + let read = self.inner.read(output)?; + if read == 0 { + self.check_eof()?; + return Ok(0); + } + self.meter(&output[..read])?; + Ok(read) + } +} + +#[cfg(test)] +mod tests { + use std::io::{Cursor, Read}; + + use super::*; + + fn header(entry_type: u8, size: u64, base_256: bool) -> [u8; 512] { + let mut header = [0_u8; 512]; + header[0] = b'x'; + header[156] = entry_type; + if base_256 { + header[124] = 0x80; + header[128..136].copy_from_slice(&size.to_be_bytes()); + } else { + let encoded = format!("{:011o}\0", size); + header[124..136].copy_from_slice(encoded.as_bytes()); + } + header + } + + fn consume(bytes: Vec, limit: u64) -> io::Result> { + let mut output = Vec::new(); + TarMetadataMeter::new(Cursor::new(bytes), limit).read_to_end(&mut output)?; + Ok(output) + } + + #[test] + fn meters_octal_and_base_256_metadata_before_forwarding() { + for base_256 in [false, true] { + let bytes = [header(b'x', 513, base_256).as_slice(), &vec![0; 1024]].concat(); + let error = consume(bytes, 512).unwrap_err(); + assert!(error.to_string().contains(META_LIMIT)); + } + } + + #[test] + fn rejects_in_limit_pax_without_interpreting_size_overrides() { + let bytes = [header(b'x', 8, false).as_slice(), &vec![0; 512]].concat(); + let error = consume(bytes, 1024).unwrap_err(); + assert!(error.to_string().contains(INVALID_HEADER)); + assert!(error.to_string().contains("PAX metadata")); + } + + #[test] + fn meters_chained_sparse_extension_blocks() { + let mut main = header(b'S', 0, false); + main[482] = 1; + let mut first = [0_u8; 512]; + first[504] = 1; + let second = [0_u8; 512]; + let error = consume([main.as_slice(), &first, &second].concat(), 512).unwrap_err(); + assert!(error.to_string().contains(META_LIMIT)); + } + + #[test] + fn rejects_random_invalid_size_headers_and_truncation() { + let mut seed = 0x1234_5678_u64; + for _ in 0..256 { + seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1); + let mut block = header(b'0', 0, false); + block[124..136].fill(((seed >> 32) as u8) | 0x08); + assert!(consume(block.to_vec(), 1024).is_err()); + } + assert!( + consume(vec![0_u8; 511], 1024) + .unwrap_err() + .to_string() + .contains(INVALID_HEADER) + ); + } +} diff --git a/native/src/unix.rs b/native/src/unix.rs new file mode 100644 index 0000000..9685632 --- /dev/null +++ b/native/src/unix.rs @@ -0,0 +1,875 @@ +#[cfg(target_os = "macos")] +use std::ffi::CString; +#[cfg(target_os = "macos")] +use std::ffi::c_void; +use std::io::{Read, Write}; +use std::os::fd::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd}; + +use rustix::fs::{AtFlags, FileType, Mode, OFlags, RenameFlags}; + +use crate::{FileIdentity, NativeResult, native_error}; + +fn borrowed(fd: i32) -> BorrowedFd<'static> { + // SAFETY: Every public operation borrows the descriptor only for the + // duration of the call. Ownership stays with Node.js. + unsafe { BorrowedFd::borrow_raw(fd) } +} + +fn os_error(error: rustix::io::Errno, operation: &str) -> napi::Error { + let code = match error { + rustix::io::Errno::EXIST => "EEXIST", + rustix::io::Errno::NOENT => "ENOENT", + rustix::io::Errno::LOOP => "ELOOP", + rustix::io::Errno::NOTDIR => "ENOTDIR", + rustix::io::Errno::ACCESS => "EACCES", + rustix::io::Errno::PERM => "EPERM", + rustix::io::Errno::XDEV => "EXDEV", + rustix::io::Errno::NOTEMPTY => "ENOTEMPTY", + _ => "EIO", + }; + native_error(code, format!("{operation}: {error}")) +} + +fn validate_beneath_path(path: &str) -> NativeResult<()> { + if path.starts_with('/') || path.split('/').any(|segment| segment == "..") { + return Err(native_error( + "EINVAL", + "relative path must remain beneath root", + )); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +pub fn open_beneath(root_fd: i32, rel_path: &str, flags: i32) -> NativeResult { + use rustix::fs::{ResolveFlags, openat2}; + + validate_beneath_path(rel_path)?; + if rel_path.is_empty() || rel_path == "." { + return rustix::io::dup(borrowed(root_fd)) + .map(OwnedFd::into_raw_fd) + .map_err(|error| os_error(error, "duplicate root descriptor")); + } + let path = if rel_path.is_empty() { "." } else { rel_path }; + let mut oflags = OFlags::from_bits_retain(flags as u32); + let require_directory = oflags.contains(OFlags::DIRECTORY); + oflags.remove(OFlags::DIRECTORY); + let mode = if oflags.intersects(OFlags::CREATE | OFlags::TMPFILE) { + Mode::from_bits_retain(0o600) + } else { + Mode::empty() + }; + let fd = openat2( + borrowed(root_fd), + path, + oflags, + mode, + ResolveFlags::BENEATH | ResolveFlags::NO_MAGICLINKS, + ) + .map_err(|error| os_error(error, "openat2 beneath root"))?; + if require_directory { + let stat = rustix::fs::fstat(fd.as_fd()) + .map_err(|error| os_error(error, "fstat opened directory"))?; + if !FileType::from_raw_mode(stat.st_mode).is_dir() { + return Err(native_error("ENOTDIR", "opened path is not a directory")); + } + } + Ok(fd.into_raw_fd()) +} + +#[cfg(target_os = "macos")] +pub fn open_beneath(root_fd: i32, rel_path: &str, flags: i32) -> NativeResult { + validate_beneath_path(rel_path)?; + macos::open_beneath(root_fd, rel_path, flags) +} + +fn split_parent(path: &str) -> NativeResult<(&str, &str)> { + match path.rsplit_once('/') { + Some((parent, basename)) if !basename.is_empty() => Ok((parent, basename)), + None if !path.is_empty() => Ok(("", path)), + _ => Err(native_error("EINVAL", "operation requires a basename")), + } +} + +fn directory_open_flags() -> i32 { + (OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC).bits() as i32 +} + +fn open_parent(root_fd: i32, path: &str) -> NativeResult<(OwnedFd, &str)> { + let (parent, basename) = split_parent(path)?; + let fd = open_beneath(root_fd, parent, directory_open_flags())?; + // SAFETY: open_beneath returns a newly owned descriptor. + Ok((unsafe { OwnedFd::from_raw_fd(fd) }, basename)) +} + +pub fn mkdir_beneath(root_fd: i32, rel_path: &str, mode: u32) -> NativeResult<()> { + if rel_path.is_empty() || rel_path == "." { + return Ok(()); + } + let mut current = rustix::io::dup(borrowed(root_fd)) + .map_err(|error| os_error(error, "duplicate root descriptor"))?; + for segment in rel_path + .split('/') + .filter(|segment| !segment.is_empty() && *segment != ".") + { + match rustix::fs::mkdirat(current.as_fd(), segment, Mode::from_bits_retain(mode as _)) { + Ok(()) | Err(rustix::io::Errno::EXIST) => {} + Err(error) => return Err(os_error(error, "mkdirat beneath root")), + } + let next = open_beneath(current.as_fd().as_raw_fd(), segment, directory_open_flags())?; + // SAFETY: open_beneath returns a newly owned descriptor. + current = unsafe { OwnedFd::from_raw_fd(next) }; + } + Ok(()) +} + +pub fn link_beneath( + source_root_fd: i32, + source_rel_path: &str, + target_root_fd: i32, + target_rel_path: &str, +) -> NativeResult<()> { + let (source_parent, source_name) = open_parent(source_root_fd, source_rel_path)?; + let (target_parent, target_name) = open_parent(target_root_fd, target_rel_path)?; + rustix::fs::linkat( + source_parent.as_fd(), + source_name, + target_parent.as_fd(), + target_name, + AtFlags::empty(), + ) + .map_err(|error| os_error(error, "linkat beneath roots")) +} + +pub fn rename_no_replace( + source_root_fd: i32, + source_rel_path: &str, + target_root_fd: i32, + target_rel_path: &str, +) -> NativeResult<()> { + let (source_parent, source_name) = open_parent(source_root_fd, source_rel_path)?; + let (target_parent, target_name) = open_parent(target_root_fd, target_rel_path)?; + let result = rustix::fs::renameat_with( + source_parent.as_fd(), + source_name, + target_parent.as_fd(), + target_name, + RenameFlags::NOREPLACE, + ); + match result { + Ok(()) => Ok(()), + Err(_error) + if rustix::fs::statat( + target_parent.as_fd(), + target_name, + AtFlags::SYMLINK_NOFOLLOW, + ) + .is_ok() => + { + Err(native_error("EEXIST", "rename destination already exists")) + } + Err(error) => Err(os_error(error, "rename without replacement")), + } +} + +pub fn fstat_identity(fd: i32) -> NativeResult { + let stat = rustix::fs::fstat(borrowed(fd)).map_err(|error| os_error(error, "fstat"))?; + let file_type = FileType::from_raw_mode(stat.st_mode); + Ok(FileIdentity { + dev: stat.st_dev as f64, + ino: stat.st_ino as f64, + mode: stat.st_mode as u32, + nlink: stat.st_nlink as f64, + size: stat.st_size as f64, + is_file: file_type.is_file(), + is_directory: file_type.is_dir(), + is_symbolic_link: file_type.is_symlink(), + }) +} + +pub fn write_archive_file( + root_fd: i32, + rel_path: &str, + reader: &mut R, + expected_size: u64, + mode: u32, +) -> NativeResult<()> { + let flags = OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::CLOEXEC; + let fd = open_beneath(root_fd, rel_path, flags.bits() as i32)?; + // SAFETY: open_beneath returned a fresh descriptor owned by this call. + let owned = unsafe { OwnedFd::from_raw_fd(fd) }; + let mut file = std::fs::File::from(owned); + let copied = std::io::copy(&mut reader.take(expected_size.saturating_add(1)), &mut file) + .map_err(|error| native_error("EIO", format!("write archive entry: {error}")))?; + if copied != expected_size { + return Err(native_error( + "EINVAL", + "archive entry size did not match its manifest", + )); + } + file.flush() + .map_err(|error| native_error("EIO", format!("flush archive entry: {error}")))?; + rustix::fs::fchmod(file.as_fd(), Mode::from_bits_retain(mode as _)) + .map_err(|error| os_error(error, "set archive entry mode")) +} + +pub fn chmod_beneath(root_fd: i32, rel_path: &str, mode: u32) -> NativeResult<()> { + let fd = open_beneath( + root_fd, + rel_path, + (OFlags::RDONLY | OFlags::CLOEXEC | OFlags::NOFOLLOW).bits() as i32, + )?; + // SAFETY: open_beneath returned a fresh descriptor owned by this call. + let owned = unsafe { OwnedFd::from_raw_fd(fd) }; + rustix::fs::fchmod(owned.as_fd(), Mode::from_bits_retain(mode as _)) + .map_err(|error| os_error(error, "set archive directory mode")) +} + +pub type IndependentReader = i32; + +pub fn open_independent_reader(fd: i32) -> NativeResult { + Ok(fd) +} + +pub fn read_at(reader: &IndependentReader, buffer: &mut [u8], offset: u64) -> NativeResult { + rustix::io::pread(borrowed(*reader), buffer, offset) + .map_err(|error| os_error(error, "read file at offset")) +} + +#[cfg(target_os = "linux")] +fn create_exclusive_target(root_fd: i32, rel_path: &str) -> NativeResult { + let flags = OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::CLOEXEC; + let fd = open_beneath(root_fd, rel_path, flags.bits() as i32)?; + // SAFETY: open_beneath returned a fresh descriptor owned by this call. + Ok(unsafe { OwnedFd::from_raw_fd(fd) }) +} + +fn remove_created_target(root_fd: i32, rel_path: &str, target: &OwnedFd) { + #[cfg(target_os = "macos")] + // SAFETY: target is an open descriptor owned by the caller. + unsafe { + libc::fchflags(target.as_raw_fd(), 0); + } + let Ok(target_stat) = rustix::fs::fstat(target.as_fd()) else { + return; + }; + let Ok((parent, name)) = open_parent(root_fd, rel_path) else { + return; + }; + let Ok(path_stat) = rustix::fs::statat(parent.as_fd(), name, AtFlags::SYMLINK_NOFOLLOW) else { + return; + }; + if target_stat.st_dev == path_stat.st_dev && target_stat.st_ino == path_stat.st_ino { + let _ = rustix::fs::unlinkat(parent.as_fd(), name, AtFlags::empty()); + } +} + +#[cfg(target_os = "linux")] +pub fn clone_file_exclusive( + source_fd: i32, + target_root_fd: i32, + target_rel_path: &str, +) -> NativeResult { + let target = create_exclusive_target(target_root_fd, target_rel_path)?; + if let Err(error) = rustix::fs::ioctl_ficlone(target.as_fd(), borrowed(source_fd)) { + remove_created_target(target_root_fd, target_rel_path, &target); + return Err(native_error( + "ENOTSUP", + format!("FICLONE is unavailable: {error}"), + )); + } + if let Err(error) = rustix::fs::fchmod(target.as_fd(), Mode::from_bits_retain(0o600)) + .and_then(|()| rustix::fs::fsync(target.as_fd())) + { + remove_created_target(target_root_fd, target_rel_path, &target); + return Err(os_error(error, "normalize cloned file")); + } + Ok(target.into_raw_fd()) +} + +#[cfg(target_os = "macos")] +pub fn clone_file_exclusive( + source_fd: i32, + target_root_fd: i32, + target_rel_path: &str, +) -> NativeResult { + use std::sync::atomic::{AtomicU64, Ordering}; + + const CLONE_NOOWNERCOPY: u32 = 0x0002; + static CLONE_COUNTER: AtomicU64 = AtomicU64::new(0); + let parent_stat = rustix::fs::fstat(borrowed(target_root_fd)) + .map_err(|error| os_error(error, "inspect clone target parent"))?; + // SAFETY: geteuid has no preconditions. + let effective_uid = unsafe { libc::geteuid() }; + if parent_stat.st_uid != effective_uid + || parent_stat.st_mode & 0o022 != 0 + || macos_acl_has_entries(target_root_fd)? + { + return Err(native_error( + "ENOTSUP", + "cloning requires an owned target parent without broad modes or ACLs", + )); + } + let source_stat = rustix::fs::fstat(borrowed(source_fd)) + .map_err(|error| os_error(error, "inspect clone source"))?; + if source_stat.st_flags != 0 { + return Err(native_error( + "ENOTSUP", + "cloning is disabled for sources with file flags", + )); + } + let stage_path = (0..8) + .find_map(|_| { + let nonce = CLONE_COUNTER.fetch_add(1, Ordering::Relaxed); + let candidate = format!(".fs-safe-clone-stage-{}-{nonce}", std::process::id()); + match rustix::fs::mkdirat( + borrowed(target_root_fd), + candidate.as_str(), + Mode::from_bits_retain(0o700), + ) { + Ok(()) => Some(candidate), + Err(rustix::io::Errno::EXIST) => None, + Err(_) => None, + } + }) + .ok_or_else(|| native_error("EIO", "create private clone staging directory"))?; + let stage_fd = match open_beneath( + target_root_fd, + &stage_path, + (OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW).bits() as i32, + ) { + Ok(fd) => { + // SAFETY: open_beneath returned a fresh descriptor owned here. + unsafe { OwnedFd::from_raw_fd(fd) } + } + Err(error) => { + let _ = rustix::fs::unlinkat( + borrowed(target_root_fd), + stage_path.as_str(), + AtFlags::REMOVEDIR, + ); + return Err(error); + } + }; + + let payload = CString::new("payload").unwrap(); + // SAFETY: descriptors are borrowed for this call and payload is NUL-terminated. + if unsafe { + libc::fclonefileat( + source_fd, + stage_fd.as_raw_fd(), + payload.as_ptr(), + CLONE_NOOWNERCOPY, + ) + } != 0 + { + let error = std::io::Error::last_os_error(); + let _ = rustix::fs::unlinkat( + borrowed(target_root_fd), + stage_path.as_str(), + AtFlags::REMOVEDIR, + ); + let code = match error.raw_os_error() { + Some(libc::EXDEV) | Some(libc::ENOTSUP) | Some(libc::EINVAL) => "ENOTSUP", + Some(libc::EACCES) => "EACCES", + Some(libc::EPERM) => "EPERM", + _ => "EIO", + }; + return Err(native_error(code, format!("fclonefileat: {error}"))); + } + let target_fd = match open_beneath( + stage_fd.as_raw_fd(), + "payload", + (OFlags::RDONLY | OFlags::CLOEXEC | OFlags::NOFOLLOW).bits() as i32, + ) { + Ok(fd) => fd, + Err(error) => { + let _ = rustix::fs::unlinkat(stage_fd.as_fd(), "payload", AtFlags::empty()); + let _ = rustix::fs::unlinkat( + borrowed(target_root_fd), + stage_path.as_str(), + AtFlags::REMOVEDIR, + ); + return Err(error); + } + }; + // SAFETY: open_beneath returned a fresh descriptor owned here. + let target = unsafe { OwnedFd::from_raw_fd(target_fd) }; + + let normalize = || -> NativeResult<()> { + // SAFETY: target is an open descriptor owned by this call. + if unsafe { libc::fchflags(target.as_raw_fd(), 0) } != 0 { + return Err(native_error( + "EIO", + format!( + "clear cloned file flags: {}", + std::io::Error::last_os_error() + ), + )); + } + clear_macos_acl(target.as_raw_fd())?; + rustix::fs::fchmod(target.as_fd(), Mode::from_bits_retain(0o600)) + .map_err(|error| os_error(error, "set cloned file mode"))?; + clear_macos_xattrs(target.as_raw_fd())?; + rustix::fs::fsync(target.as_fd()).map_err(|error| os_error(error, "sync cloned file")) + }; + if let Err(error) = normalize() { + remove_created_target(stage_fd.as_raw_fd(), "payload", &target); + let _ = rustix::fs::unlinkat( + borrowed(target_root_fd), + stage_path.as_str(), + AtFlags::REMOVEDIR, + ); + return Err(error); + } + if let Err(error) = rename_no_replace( + stage_fd.as_raw_fd(), + "payload", + target_root_fd, + target_rel_path, + ) { + remove_created_target(stage_fd.as_raw_fd(), "payload", &target); + let _ = rustix::fs::unlinkat( + borrowed(target_root_fd), + stage_path.as_str(), + AtFlags::REMOVEDIR, + ); + return Err(error); + } + let _ = rustix::fs::unlinkat( + borrowed(target_root_fd), + stage_path.as_str(), + AtFlags::REMOVEDIR, + ); + Ok(target.into_raw_fd()) +} + +#[cfg(target_os = "macos")] +fn clear_macos_acl(fd: i32) -> NativeResult<()> { + const ACL_TYPE_EXTENDED: i32 = 0x0000_0100; + unsafe extern "C" { + fn acl_init(count: i32) -> *mut c_void; + fn acl_set_fd_np(fd: i32, acl: *mut c_void, acl_type: i32) -> i32; + fn acl_free(object: *mut c_void) -> i32; + } + + // SAFETY: acl_init allocates an empty ACL owned by this function. + let acl = unsafe { acl_init(0) }; + if acl.is_null() { + return Err(native_error( + "EIO", + format!("allocate empty ACL: {}", std::io::Error::last_os_error()), + )); + } + // SAFETY: fd and acl are valid for the duration of the call. + let result = unsafe { acl_set_fd_np(fd, acl, ACL_TYPE_EXTENDED) }; + // SAFETY: acl was allocated by acl_init and is freed exactly once. + unsafe { acl_free(acl) }; + if result != 0 { + return Err(native_error( + "EIO", + format!("clear cloned file ACL: {}", std::io::Error::last_os_error()), + )); + } + Ok(()) +} + +#[cfg(target_os = "macos")] +fn macos_acl_has_entries(fd: i32) -> NativeResult { + const ACL_TYPE_EXTENDED: i32 = 0x0000_0100; + const ACL_FIRST_ENTRY: i32 = 0; + unsafe extern "C" { + fn acl_get_fd_np(fd: i32, acl_type: i32) -> *mut c_void; + fn acl_get_entry(acl: *mut c_void, entry_id: i32, entry: *mut *mut c_void) -> i32; + fn acl_free(object: *mut c_void) -> i32; + } + + // SAFETY: acl_get_fd_np borrows fd and returns an owned ACL object. + let acl = unsafe { acl_get_fd_np(fd, ACL_TYPE_EXTENDED) }; + if acl.is_null() { + let error = std::io::Error::last_os_error(); + let code = error.raw_os_error(); + return if matches!(code, Some(libc::ENOENT) | Some(libc::ENOATTR)) { + Ok(false) + } else if code == Some(libc::ENOTSUP) + || code == Some(libc::EOPNOTSUPP) + || code == Some(libc::EINVAL) + { + Err(native_error( + "ENOTSUP", + format!("inspect parent ACL: {error}"), + )) + } else { + Err(native_error("EIO", format!("inspect parent ACL: {error}"))) + }; + } + let mut entry = std::ptr::null_mut(); + // SAFETY: acl is valid and entry is writable for one pointer. + let result = unsafe { acl_get_entry(acl, ACL_FIRST_ENTRY, &mut entry) }; + // SAFETY: acl was returned by acl_get_fd_np and is freed exactly once. + unsafe { acl_free(acl) }; + match result { + 1 => Ok(true), + 0 => Ok(false), + _ => Err(native_error( + "EIO", + format!("enumerate parent ACL: {}", std::io::Error::last_os_error()), + )), + } +} + +#[cfg(target_os = "macos")] +fn clear_macos_xattrs(fd: i32) -> NativeResult<()> { + // SAFETY: null buffer queries the required list size. + let length = unsafe { libc::flistxattr(fd, std::ptr::null_mut(), 0, 0) }; + if length < 0 { + return Err(native_error( + "EIO", + format!( + "list cloned file xattrs: {}", + std::io::Error::last_os_error() + ), + )); + } + if length == 0 { + return Ok(()); + } + let mut names = vec![0_u8; length as usize]; + // SAFETY: names is writable for its full allocation. + let read = unsafe { libc::flistxattr(fd, names.as_mut_ptr().cast(), names.len(), 0) }; + if read < 0 { + return Err(native_error( + "EIO", + format!( + "read cloned file xattrs: {}", + std::io::Error::last_os_error() + ), + )); + } + names.truncate(read as usize); + for name in names + .split(|byte| *byte == 0) + .filter(|name| !name.is_empty()) + { + let name = CString::new(name) + .map_err(|_| native_error("EINVAL", "cloned xattr name contains a NUL byte"))?; + // SAFETY: name is NUL-terminated and fd remains open. + if unsafe { libc::fremovexattr(fd, name.as_ptr(), 0) } != 0 + && std::io::Error::last_os_error().raw_os_error() != Some(libc::ENOATTR) + { + return Err(native_error( + "EIO", + format!( + "remove cloned file xattr: {}", + std::io::Error::last_os_error() + ), + )); + } + } + Ok(()) +} + +#[cfg(target_os = "linux")] +pub fn copy_file_range_exclusive( + source_fd: i32, + target_root_fd: i32, + target_rel_path: &str, +) -> NativeResult<(i32, u64)> { + let target = create_exclusive_target(target_root_fd, target_rel_path)?; + let source_stat = rustix::fs::fstat(borrowed(source_fd)) + .map_err(|error| os_error(error, "inspect copy source"))?; + let expected = u64::try_from(source_stat.st_size) + .map_err(|_| native_error("EINVAL", "copy source has a negative size"))?; + let mut source_offset = 0_u64; + let mut target_offset = 0_u64; + while source_offset < expected { + let length = usize::try_from((expected - source_offset).min(16 * 1024 * 1024)).unwrap(); + match rustix::fs::copy_file_range( + borrowed(source_fd), + Some(&mut source_offset), + target.as_fd(), + Some(&mut target_offset), + length, + ) { + Ok(0) => { + remove_created_target(target_root_fd, target_rel_path, &target); + return Err(native_error("EIO", "copy_file_range made no progress")); + } + Ok(_) => {} + Err(error) => { + remove_created_target(target_root_fd, target_rel_path, &target); + return Err(native_error( + "ENOTSUP", + format!("copy_file_range is unavailable: {error}"), + )); + } + } + } + if let Err(error) = rustix::fs::fchmod(target.as_fd(), Mode::from_bits_retain(0o600)) + .and_then(|()| rustix::fs::fsync(target.as_fd())) + { + remove_created_target(target_root_fd, target_rel_path, &target); + return Err(os_error(error, "normalize copied file")); + } + Ok((target.into_raw_fd(), target_offset)) +} + +#[cfg(target_os = "macos")] +pub fn copy_file_range_exclusive( + _source_fd: i32, + _target_root_fd: i32, + _target_rel_path: &str, +) -> NativeResult<(i32, u64)> { + Err(native_error( + "ENOTSUP", + "copy_file_range is only available on Linux", + )) +} + +#[cfg(target_os = "macos")] +mod macos { + use std::collections::VecDeque; + use std::ffi::{CStr, CString}; + use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; + + use crate::{NativeResult, native_error}; + + const MAX_SYMLINKS: usize = 40; + + fn last_error(operation: &str) -> napi::Error { + let error = std::io::Error::last_os_error(); + let code = match error.raw_os_error() { + Some(libc::EEXIST) => "EEXIST", + Some(libc::ENOENT) => "ENOENT", + Some(libc::ELOOP) => "ELOOP", + Some(libc::ENOTDIR) => "ENOTDIR", + Some(libc::EACCES) => "EACCES", + Some(libc::EPERM) => "EPERM", + _ => "EIO", + }; + native_error(code, format!("{operation}: {error}")) + } + + fn duplicate(fd: RawFd) -> NativeResult { + // SAFETY: dup does not borrow beyond this call and returns a fresh fd. + let duplicated = unsafe { libc::dup(fd) }; + if duplicated < 0 { + return Err(last_error("duplicate root descriptor")); + } + // SAFETY: duplicated is a new owned descriptor. + Ok(unsafe { OwnedFd::from_raw_fd(duplicated) }) + } + + fn root_path(fd: RawFd) -> NativeResult { + let mut buffer = vec![0_i8; libc::PATH_MAX as usize]; + // SAFETY: buffer is writable for PATH_MAX bytes. + if unsafe { libc::fcntl(fd, libc::F_GETPATH, buffer.as_mut_ptr()) } < 0 { + return Err(last_error("resolve root descriptor path")); + } + // SAFETY: F_GETPATH writes a NUL-terminated string on success. + Ok(unsafe { CStr::from_ptr(buffer.as_ptr()) } + .to_string_lossy() + .into_owned()) + } + + fn read_link(fd: RawFd, name: &CString) -> NativeResult { + let mut buffer = vec![0_u8; libc::PATH_MAX as usize]; + // SAFETY: pointers are valid for this call and the buffer is writable. + let read = unsafe { + libc::readlinkat(fd, name.as_ptr(), buffer.as_mut_ptr().cast(), buffer.len()) + }; + if read < 0 { + return Err(last_error("read symlink beneath root")); + } + buffer.truncate(read as usize); + String::from_utf8(buffer) + .map_err(|_| native_error("EINVAL", "symlink target is not valid UTF-8")) + } + + fn normalize(mut base: Vec, target: &str) -> NativeResult> { + for segment in target.split('/') { + match segment { + "" | "." => {} + ".." => { + if base.pop().is_none() { + return Err(native_error("EXDEV", "symlink target escapes root")); + } + } + value => base.push(value.to_owned()), + } + } + Ok(base) + } + + fn absolute_target_segments(root_fd: RawFd, target: &str) -> NativeResult> { + let root = root_path(root_fd)?; + let relative = target + .strip_prefix(&root) + .and_then(|value| { + value + .strip_prefix('/') + .or(Some(value)) + .filter(|_| target == root || target.as_bytes().get(root.len()) == Some(&b'/')) + }) + .ok_or_else(|| native_error("EXDEV", "absolute symlink target escapes root"))?; + normalize(Vec::new(), relative) + } + + pub fn open_beneath(root_fd: RawFd, rel_path: &str, flags: i32) -> NativeResult { + if rel_path.is_empty() || rel_path == "." { + return Ok(duplicate(root_fd)?.into_raw_fd()); + } + let mut queue: VecDeque = rel_path + .split('/') + .filter(|segment| !segment.is_empty() && *segment != ".") + .map(ToOwned::to_owned) + .collect(); + let mut current = duplicate(root_fd)?; + let mut logical: Vec = Vec::new(); + let mut followed = 0; + + while let Some(segment) = queue.pop_front() { + let name = CString::new(segment.as_bytes()) + .map_err(|_| native_error("EINVAL", "path segment contains a NUL byte"))?; + let is_final = queue.is_empty(); + let open_flags = if is_final { + flags | libc::O_CLOEXEC | libc::O_NOFOLLOW + } else { + libc::O_RDONLY | libc::O_DIRECTORY | libc::O_CLOEXEC | libc::O_NOFOLLOW + }; + // SAFETY: current and name stay valid for the duration of openat. + let opened = + unsafe { libc::openat(current.as_raw_fd(), name.as_ptr(), open_flags, 0o600) }; + if opened >= 0 { + if is_final { + return Ok(opened); + } + // SAFETY: opened is a new owned directory descriptor. + current = unsafe { OwnedFd::from_raw_fd(opened) }; + logical.push(segment); + continue; + } + let error = std::io::Error::last_os_error(); + let errno = error.raw_os_error(); + if !matches!(errno, Some(libc::ELOOP) | Some(libc::ENOTDIR)) + || (is_final && flags & libc::O_NOFOLLOW != 0) + { + return Err(last_error("open path beneath root")); + } + followed += 1; + if followed > MAX_SYMLINKS { + return Err(native_error("ELOOP", "too many symlinks beneath root")); + } + let target = match read_link(current.as_raw_fd(), &name) { + Ok(target) => target, + Err(_) => { + let code = if errno == Some(libc::ENOTDIR) { + "ENOTDIR" + } else { + "ELOOP" + }; + return Err(native_error( + code, + format!("open path beneath root: {error}"), + )); + } + }; + let resolved = if target.starts_with('/') { + absolute_target_segments(root_fd, &target)? + } else { + normalize(logical.clone(), &target)? + }; + let remainder: Vec = queue.drain(..).collect(); + queue = resolved.into_iter().chain(remainder).collect(); + current = duplicate(root_fd)?; + logical.clear(); + } + Err(native_error("EINVAL", "path did not resolve to an entry")) + } +} + +#[cfg(test)] +mod tests { + use std::fs::{self, OpenOptions}; + use std::os::fd::{AsRawFd, FromRawFd}; + use std::time::{SystemTime, UNIX_EPOCH}; + + use super::*; + + fn temp_root(label: &str) -> std::path::PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "fs-safe-native-{label}-{}-{nonce}", + std::process::id() + )); + fs::create_dir(&path).unwrap(); + path + } + + #[test] + fn opens_and_creates_only_beneath_root() { + let root = temp_root("open"); + fs::create_dir(root.join("nested")).unwrap(); + fs::write(root.join("nested/file"), b"ok").unwrap(); + let root_handle = OpenOptions::new().read(true).open(&root).unwrap(); + let fd = open_beneath( + root_handle.as_raw_fd(), + "nested/file", + OFlags::RDONLY.bits() as i32, + ) + .unwrap(); + // SAFETY: fd is uniquely owned after open_beneath. + let file = unsafe { std::fs::File::from_raw_fd(fd) }; + assert_eq!(fstat_identity(file.as_raw_fd()).unwrap().size, 2.0); + assert!( + open_beneath( + root_handle.as_raw_fd(), + "../outside", + OFlags::RDONLY.bits() as i32, + ) + .is_err() + ); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn rename_no_replace_preserves_existing_target() { + let root = temp_root("rename"); + fs::write(root.join("source"), b"source").unwrap(); + fs::write(root.join("target"), b"target").unwrap(); + let root_handle = OpenOptions::new().read(true).open(&root).unwrap(); + let error = rename_no_replace( + root_handle.as_raw_fd(), + "source", + root_handle.as_raw_fd(), + "target", + ) + .unwrap_err(); + assert_eq!(error.status, "EEXIST"); + assert_eq!(fs::read(root.join("target")).unwrap(), b"target"); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(target_os = "macos")] + #[test] + fn follows_in_root_symlink_by_re_resolving_from_root() { + use std::os::unix::fs::symlink; + let root = temp_root("symlink"); + fs::create_dir(root.join("real")).unwrap(); + fs::write(root.join("real/file"), b"ok").unwrap(); + symlink("real", root.join("alias")).unwrap(); + let root_handle = OpenOptions::new().read(true).open(&root).unwrap(); + let fd = open_beneath( + root_handle.as_raw_fd(), + "alias/file", + OFlags::RDONLY.bits() as i32, + ) + .unwrap(); + // SAFETY: fd is uniquely owned after open_beneath. + drop(unsafe { std::fs::File::from_raw_fd(fd) }); + fs::remove_dir_all(root).unwrap(); + } +} diff --git a/native/src/windows.rs b/native/src/windows.rs new file mode 100644 index 0000000..95e12c5 --- /dev/null +++ b/native/src/windows.rs @@ -0,0 +1,704 @@ +use std::ffi::c_void; +use std::io::{Read, Write}; +use std::mem::{size_of, zeroed}; +use std::os::windows::ffi::OsStrExt; +use std::os::windows::io::FromRawHandle; +use std::ptr::{null, null_mut}; + +use windows_sys::Win32::Foundation::{ + CloseHandle, ERROR_ACCESS_DENIED, ERROR_ALREADY_EXISTS, ERROR_FILE_EXISTS, + ERROR_FILE_NOT_FOUND, ERROR_PATH_NOT_FOUND, GENERIC_READ, GetLastError, HANDLE, + INVALID_HANDLE_VALUE, +}; +use windows_sys::Win32::Storage::FileSystem::{ + BY_HANDLE_FILE_INFORMATION, FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_REPARSE_POINT, + FILE_ATTRIBUTE_TAG_INFO, FILE_GENERIC_READ, FILE_GENERIC_WRITE, FILE_SHARE_DELETE, + FILE_SHARE_READ, FILE_SHARE_WRITE, FileAttributeTagInfo, GetFileInformationByHandle, + GetFileInformationByHandleEx, ReOpenFile, +}; +use windows_sys::Win32::System::IO::IO_STATUS_BLOCK; +use windows_sys::Win32::System::LibraryLoader::{GetModuleHandleW, GetProcAddress}; + +use crate::{FileIdentity, NativeResult, native_error}; + +const O_WRONLY: i32 = 0x0001; +const O_RDWR: i32 = 0x0002; +const O_APPEND: i32 = 0x0008; +const O_CREAT: i32 = 0x0100; +const O_TRUNC: i32 = 0x0200; +const O_EXCL: i32 = 0x0400; +const O_BINARY: i32 = 0x8000; + +const DELETE_ACCESS: u32 = 0x0001_0000; +const SYNCHRONIZE_ACCESS: u32 = 0x0010_0000; +const FILE_READ_ATTRIBUTES: u32 = 0x0000_0080; +const FILE_WRITE_ATTRIBUTES: u32 = 0x0000_0100; +const FILE_OPEN: u32 = 1; +const FILE_CREATE: u32 = 2; +const FILE_OPEN_IF: u32 = 3; +const FILE_OVERWRITE: u32 = 4; +const FILE_OVERWRITE_IF: u32 = 5; +const FILE_DIRECTORY_FILE: u32 = 0x0000_0001; +const FILE_SYNCHRONOUS_IO_NONALERT: u32 = 0x0000_0020; +const FILE_NON_DIRECTORY_FILE: u32 = 0x0000_0040; +const FILE_OPEN_REPARSE_POINT: u32 = 0x0020_0000; +const OBJ_CASE_INSENSITIVE: u32 = 0x0000_0040; +const OBJ_DONT_REPARSE: u32 = 0x0000_1000; +const FILE_RENAME_FLAG_POSIX_SEMANTICS: u32 = 0x0000_0002; +const FILE_LINK_INFORMATION_CLASS: i32 = 11; +const FILE_RENAME_INFORMATION_EX_CLASS: i32 = 65; + +#[repr(C)] +struct UnicodeString { + length: u16, + maximum_length: u16, + buffer: *mut u16, +} + +#[repr(C)] +struct ObjectAttributes { + length: u32, + root_directory: HANDLE, + object_name: *mut UnicodeString, + attributes: u32, + security_descriptor: *mut c_void, + security_quality_of_service: *mut c_void, +} + +#[link(name = "ntdll")] +unsafe extern "system" { + fn NtCreateFile( + file_handle: *mut HANDLE, + desired_access: u32, + object_attributes: *mut ObjectAttributes, + io_status_block: *mut IO_STATUS_BLOCK, + allocation_size: *const i64, + file_attributes: u32, + share_access: u32, + create_disposition: u32, + create_options: u32, + ea_buffer: *const c_void, + ea_length: u32, + ) -> i32; + fn NtSetInformationFile( + file_handle: HANDLE, + io_status_block: *mut IO_STATUS_BLOCK, + file_information: *const c_void, + length: u32, + file_information_class: i32, + ) -> i32; + fn NtReadFile( + file_handle: HANDLE, + event: HANDLE, + apc_routine: *const c_void, + apc_context: *const c_void, + io_status_block: *mut IO_STATUS_BLOCK, + buffer: *mut c_void, + length: u32, + byte_offset: *const i64, + key: *const u32, + ) -> i32; + fn RtlNtStatusToDosError(status: i32) -> u32; +} + +#[link(name = "msvcrt")] +unsafe extern "C" { + fn _get_osfhandle(fd: i32) -> isize; + fn _open_osfhandle(handle: isize, flags: i32) -> i32; + fn _set_thread_local_invalid_parameter_handler( + handler: InvalidParameterHandler, + ) -> InvalidParameterHandler; +} + +type InvalidParameterHandler = Option< + unsafe extern "C" fn( + expression: *const u16, + function: *const u16, + file: *const u16, + line: u32, + reserved: usize, + ), +>; + +unsafe extern "C" fn ignore_invalid_parameter( + _expression: *const u16, + _function: *const u16, + _file: *const u16, + _line: u32, + _reserved: usize, +) { +} + +struct OwnedHandle(HANDLE); + +impl Drop for OwnedHandle { + fn drop(&mut self) { + if !self.0.is_null() && self.0 != INVALID_HANDLE_VALUE { + // SAFETY: this wrapper uniquely owns the handle. + unsafe { CloseHandle(self.0) }; + } + } +} + +impl OwnedHandle { + fn into_raw(mut self) -> HANDLE { + let handle = self.0; + self.0 = null_mut(); + handle + } +} + +fn root_handle(fd: i32) -> NativeResult { + let direct = fd as usize as HANDLE; + // Node/libuv may expose a Windows HANDLE directly rather than a UCRT fd. + // Probe that representation first with a non-destructive handle query. + let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { zeroed() }; + if !direct.is_null() && unsafe { GetFileInformationByHandle(direct, &mut info) } != 0 { + return Ok(direct); + } + // Node's Windows fs descriptors are owned by libuv and are not guaranteed + // to belong to the addon's CRT table. Resolve libuv's public conversion + // function from the Node executable before trying the CRT fallback. + let node_module = unsafe { GetModuleHandleW(null()) }; + if !node_module.is_null() { + let proc = unsafe { GetProcAddress(node_module, c"uv_get_osfhandle".as_ptr().cast()) }; + if let Some(proc) = proc { + // SAFETY: uv_get_osfhandle is exported with the documented + // `intptr_t uv_get_osfhandle(int)` signature. + let get_uv_handle: unsafe extern "C" fn(i32) -> isize = + unsafe { std::mem::transmute(proc) }; + let handle = unsafe { get_uv_handle(fd) }; + if handle != -1 { + return Ok(handle as HANDLE); + } + } + } + // _get_osfhandle invokes UCRT's invalid-parameter handler for a non-CRT + // descriptor. Override it on this thread so an invalid representation + // becomes a normal -1 result instead of terminating Node. + let previous = + unsafe { _set_thread_local_invalid_parameter_handler(Some(ignore_invalid_parameter)) }; + let handle = unsafe { _get_osfhandle(fd) }; + unsafe { _set_thread_local_invalid_parameter_handler(previous) }; + if handle == -1 { + return Err(native_error("EBADF", "invalid root file descriptor")); + } + Ok(handle as HANDLE) +} + +fn node_fd_from_handle(handle: OwnedHandle, flags: i32) -> NativeResult { + let node_module = unsafe { GetModuleHandleW(null()) }; + if !node_module.is_null() { + let proc = unsafe { GetProcAddress(node_module, c"uv_open_osfhandle".as_ptr().cast()) }; + if let Some(proc) = proc { + // SAFETY: uv_open_osfhandle is exported with the documented + // `int uv_open_osfhandle(intptr_t)` signature. + let open_uv_handle: unsafe extern "C" fn(isize) -> i32 = + unsafe { std::mem::transmute(proc) }; + let fd = unsafe { open_uv_handle(handle.0 as isize) }; + if fd >= 0 { + let _ = handle.into_raw(); + return Ok(fd); + } + } + } + let fd = unsafe { _open_osfhandle(handle.0 as isize, flags | O_BINARY | (flags & O_APPEND)) }; + if fd < 0 { + return Err(native_error( + "EIO", + "convert Windows handle to file descriptor", + )); + } + let _ = handle.into_raw(); + Ok(fd) +} + +fn wide_relative(path: &str) -> NativeResult> { + let normalized = path.replace('/', "\\"); + let wide: Vec = std::ffi::OsStr::new(&normalized).encode_wide().collect(); + if wide.len() > (u16::MAX as usize / 2) { + return Err(native_error("ENAMETOOLONG", "relative path is too long")); + } + Ok(wide) +} + +fn win_error(code: u32, operation: &str) -> napi::Error { + let typed = match code { + ERROR_FILE_EXISTS | ERROR_ALREADY_EXISTS => "EEXIST", + ERROR_FILE_NOT_FOUND | ERROR_PATH_NOT_FOUND => "ENOENT", + ERROR_ACCESS_DENIED => "EACCES", + _ => "EIO", + }; + native_error( + typed, + format!("{operation} failed with Windows error {code}"), + ) +} + +fn nt_error(status: i32, operation: &str) -> napi::Error { + // SAFETY: converting an NTSTATUS does not dereference application memory. + win_error(unsafe { RtlNtStatusToDosError(status) }, operation) +} + +fn assert_not_reparse(handle: HANDLE) -> NativeResult<()> { + // SAFETY: info is a valid output buffer for the supplied class. + let mut info: FILE_ATTRIBUTE_TAG_INFO = unsafe { zeroed() }; + let ok = unsafe { + GetFileInformationByHandleEx( + handle, + FileAttributeTagInfo, + (&mut info as *mut FILE_ATTRIBUTE_TAG_INFO).cast(), + size_of::() as u32, + ) + }; + if ok == 0 { + // SAFETY: GetLastError has no memory safety preconditions. + return Err(win_error(unsafe { GetLastError() }, "inspect opened path")); + } + if info.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT != 0 { + return Err(native_error( + "ELOOP", + "reparse points are not allowed beneath root", + )); + } + Ok(()) +} + +fn nt_open_relative( + root: HANDLE, + path: &str, + desired_access: u32, + disposition: u32, + options: u32, +) -> NativeResult { + let mut name = wide_relative(path)?; + let mut unicode = UnicodeString { + length: (name.len() * 2) as u16, + maximum_length: (name.len() * 2) as u16, + buffer: name.as_mut_ptr(), + }; + let mut attributes = ObjectAttributes { + length: size_of::() as u32, + root_directory: root, + object_name: &mut unicode, + attributes: OBJ_CASE_INSENSITIVE | OBJ_DONT_REPARSE, + security_descriptor: null_mut(), + security_quality_of_service: null_mut(), + }; + // SAFETY: all pointers reference initialized, call-scoped storage. + let mut io: IO_STATUS_BLOCK = unsafe { zeroed() }; + let mut handle: HANDLE = null_mut(); + let status = unsafe { + NtCreateFile( + &mut handle, + desired_access | FILE_READ_ATTRIBUTES | SYNCHRONIZE_ACCESS, + &mut attributes, + &mut io, + null(), + 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + disposition, + options | FILE_SYNCHRONOUS_IO_NONALERT | FILE_OPEN_REPARSE_POINT, + null(), + 0, + ) + }; + if status < 0 { + return Err(nt_error(status, "open path relative to root handle")); + } + let owned = OwnedHandle(handle); + assert_not_reparse(owned.0)?; + Ok(owned) +} + +fn access_from_flags(flags: i32) -> u32 { + let mut access = FILE_READ_ATTRIBUTES; + match flags & 3 { + O_WRONLY => access |= FILE_GENERIC_WRITE, + O_RDWR => access |= FILE_GENERIC_READ | FILE_GENERIC_WRITE, + _ => access |= FILE_GENERIC_READ, + } + access +} + +fn disposition_from_flags(flags: i32) -> u32 { + match ( + flags & O_CREAT != 0, + flags & O_EXCL != 0, + flags & O_TRUNC != 0, + ) { + (true, true, _) => FILE_CREATE, + (true, false, true) => FILE_OVERWRITE_IF, + (true, false, false) => FILE_OPEN_IF, + (false, _, true) => FILE_OVERWRITE, + _ => FILE_OPEN, + } +} + +pub fn open_beneath(root_fd: i32, rel_path: &str, flags: i32) -> NativeResult { + let handle = nt_open_relative( + root_handle(root_fd)?, + if rel_path.is_empty() { "." } else { rel_path }, + access_from_flags(flags), + disposition_from_flags(flags), + 0, + )?; + node_fd_from_handle(handle, flags) +} + +pub fn mkdir_beneath(root_fd: i32, rel_path: &str, _mode: u32) -> NativeResult<()> { + if rel_path.is_empty() || rel_path == "." { + return Ok(()); + } + let mut owned_parent: Option = None; + for segment in rel_path + .split(['/', '\\']) + .filter(|segment| !segment.is_empty() && *segment != ".") + { + let parent = owned_parent + .as_ref() + .map_or(root_handle(root_fd)?, |handle| handle.0); + owned_parent = Some(nt_open_relative( + parent, + segment, + FILE_GENERIC_READ | FILE_GENERIC_WRITE, + FILE_OPEN_IF, + FILE_DIRECTORY_FILE, + )?); + } + Ok(()) +} + +#[repr(C)] +struct FileNameInfoHeader { + flags: u32, + root_directory: HANDLE, + file_name_length: u32, +} + +#[repr(C)] +struct FileLinkInfoHeader { + replace_if_exists: u8, + root_directory: HANDLE, + file_name_length: u32, +} + +const FILE_NAME_OFFSET: usize = 20; + +fn aligned_name_buffer(byte_len: usize) -> Vec { + let word_len = byte_len.div_ceil(size_of::()); + vec![0_usize; word_len] +} + +fn set_rename_information( + source: HANDLE, + target_root: HANDLE, + target_path: &str, + operation: &str, +) -> NativeResult<()> { + let name = wide_relative(target_path)?; + let name_bytes = std::mem::size_of_val(name.as_slice()); + let byte_len = FILE_NAME_OFFSET + name_bytes; + let mut buffer = aligned_name_buffer(byte_len); + // SAFETY: the zeroed usize storage is suitably aligned, the fixed fields + // end at offset 20 on the supported Windows x64 ABI, and the allocation is + // large enough for the trailing UTF-16 filename. + unsafe { + let header = buffer.as_mut_ptr().cast::(); + (*header).flags = FILE_RENAME_FLAG_POSIX_SEMANTICS; + (*header).root_directory = target_root; + (*header).file_name_length = name_bytes as u32; + std::ptr::copy_nonoverlapping( + name.as_ptr().cast::(), + buffer.as_mut_ptr().cast::().add(FILE_NAME_OFFSET), + name_bytes, + ); + } + // SAFETY: buffer contains FILE_RENAME_INFORMATION_EX followed by the + // UTF-16 target name, and io remains valid for the synchronous call. + let mut io: IO_STATUS_BLOCK = unsafe { zeroed() }; + let status = unsafe { + NtSetInformationFile( + source, + &mut io, + buffer.as_ptr().cast(), + byte_len as u32, + FILE_RENAME_INFORMATION_EX_CLASS, + ) + }; + if status < 0 { + if nt_open_relative(target_root, target_path, FILE_READ_ATTRIBUTES, FILE_OPEN, 0).is_ok() { + return Err(native_error("EEXIST", "rename destination already exists")); + } + return Err(nt_error(status, operation)); + } + Ok(()) +} + +fn set_link_information( + source: HANDLE, + target_root: HANDLE, + target_path: &str, +) -> NativeResult<()> { + let name = wide_relative(target_path)?; + let name_bytes = std::mem::size_of_val(name.as_slice()); + let byte_len = FILE_NAME_OFFSET + name_bytes; + let mut buffer = aligned_name_buffer(byte_len); + // SAFETY: FILE_LINK_INFORMATION uses the same x64 filename offset. + unsafe { + let header = buffer.as_mut_ptr().cast::(); + (*header).replace_if_exists = 0; + (*header).root_directory = target_root; + (*header).file_name_length = name_bytes as u32; + std::ptr::copy_nonoverlapping( + name.as_ptr().cast::(), + buffer.as_mut_ptr().cast::().add(FILE_NAME_OFFSET), + name_bytes, + ); + } + // SAFETY: buffer contains FILE_LINK_INFORMATION followed by the UTF-16 + // name, and io remains valid for the synchronous call. + let mut io: IO_STATUS_BLOCK = unsafe { zeroed() }; + let status = unsafe { + NtSetInformationFile( + source, + &mut io, + buffer.as_ptr().cast(), + byte_len as u32, + FILE_LINK_INFORMATION_CLASS, + ) + }; + if status < 0 { + return Err(nt_error(status, "create hard link without replacement")); + } + Ok(()) +} + +fn open_source_for_metadata( + root_fd: i32, + path: &str, + extra_access: u32, +) -> NativeResult { + nt_open_relative( + root_handle(root_fd)?, + path, + FILE_READ_ATTRIBUTES | extra_access, + FILE_OPEN, + FILE_NON_DIRECTORY_FILE, + ) +} + +pub fn link_beneath( + source_root_fd: i32, + source_rel_path: &str, + target_root_fd: i32, + target_rel_path: &str, +) -> NativeResult<()> { + let source = open_source_for_metadata(source_root_fd, source_rel_path, FILE_WRITE_ATTRIBUTES)?; + set_link_information(source.0, root_handle(target_root_fd)?, target_rel_path) +} + +pub fn rename_no_replace( + source_root_fd: i32, + source_rel_path: &str, + target_root_fd: i32, + target_rel_path: &str, +) -> NativeResult<()> { + let source = open_source_for_metadata(source_root_fd, source_rel_path, DELETE_ACCESS)?; + set_rename_information( + source.0, + root_handle(target_root_fd)?, + target_rel_path, + "rename without replacement", + ) +} + +pub fn fstat_identity(fd: i32) -> NativeResult { + let handle = root_handle(fd)?; + assert_not_reparse(handle)?; + // SAFETY: info is a valid output buffer for this API. + let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { zeroed() }; + if unsafe { GetFileInformationByHandle(handle, &mut info) } == 0 { + // SAFETY: GetLastError has no memory safety preconditions. + return Err(win_error( + unsafe { GetLastError() }, + "inspect file identity", + )); + } + let is_directory = info.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY != 0; + let size = ((info.nFileSizeHigh as u64) << 32) | info.nFileSizeLow as u64; + let ino = ((info.nFileIndexHigh as u64) << 32) | info.nFileIndexLow as u64; + Ok(FileIdentity { + dev: info.dwVolumeSerialNumber as f64, + ino: ino as f64, + mode: if is_directory { 0o040000 } else { 0o100000 }, + nlink: info.nNumberOfLinks as f64, + size: size as f64, + is_file: !is_directory, + is_directory, + is_symbolic_link: false, + }) +} + +pub fn write_archive_file( + root_fd: i32, + rel_path: &str, + reader: &mut R, + expected_size: u64, + _mode: u32, +) -> NativeResult<()> { + let handle = nt_open_relative( + root_handle(root_fd)?, + rel_path, + FILE_GENERIC_WRITE, + FILE_CREATE, + FILE_NON_DIRECTORY_FILE, + )?; + // SAFETY: into_raw transfers the uniquely owned HANDLE to File. + let mut file = unsafe { std::fs::File::from_raw_handle(handle.into_raw()) }; + let copied = std::io::copy(&mut reader.take(expected_size.saturating_add(1)), &mut file) + .map_err(|error| native_error("EIO", format!("write archive entry: {error}")))?; + if copied != expected_size { + return Err(native_error( + "EINVAL", + "archive entry size did not match its manifest", + )); + } + file.flush() + .map_err(|error| native_error("EIO", format!("flush archive entry: {error}"))) +} + +pub fn chmod_beneath(_root_fd: i32, _rel_path: &str, _mode: u32) -> NativeResult<()> { + Ok(()) +} + +pub struct IndependentReader(OwnedHandle); + +pub fn open_independent_reader(fd: i32) -> NativeResult { + let handle = unsafe { + ReOpenFile( + root_handle(fd)?, + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + 0, + ) + }; + if handle == INVALID_HANDLE_VALUE { + return Err(win_error( + unsafe { GetLastError() }, + "reopen file for position-independent read", + )); + } + Ok(IndependentReader(OwnedHandle(handle))) +} + +pub fn read_at(reader: &IndependentReader, buffer: &mut [u8], offset: u64) -> NativeResult { + const STATUS_END_OF_FILE: i32 = 0xC000_0011_u32 as i32; + let offset = i64::try_from(offset) + .map_err(|_| native_error("EINVAL", "read offset exceeds Windows range"))?; + let length = u32::try_from(buffer.len()) + .map_err(|_| native_error("EINVAL", "read buffer exceeds Windows range"))?; + let mut io: IO_STATUS_BLOCK = unsafe { zeroed() }; + // SAFETY: the handle is valid, buffer is writable for length bytes, and + // the synchronous handle keeps all stack arguments live until completion. + let status = unsafe { + NtReadFile( + reader.0.0, + null_mut(), + null(), + null(), + &mut io, + buffer.as_mut_ptr().cast(), + length, + &offset, + null(), + ) + }; + if status == STATUS_END_OF_FILE { + return Ok(0); + } + if status < 0 { + return Err(nt_error(status, "read file at offset")); + } + Ok(io.Information) +} + +pub fn clone_file_exclusive( + _source_fd: i32, + _target_root_fd: i32, + _target_rel_path: &str, +) -> NativeResult { + Err(native_error( + "ENOTSUP", + "file cloning is not available on Windows", + )) +} + +pub fn copy_file_range_exclusive( + _source_fd: i32, + _target_root_fd: i32, + _target_rel_path: &str, +) -> NativeResult<(i32, u64)> { + Err(native_error( + "ENOTSUP", + "copy_file_range is not available on Windows", + )) +} + +#[cfg(test)] +mod tests { + use std::fs::{self, OpenOptions}; + use std::os::windows::fs::OpenOptionsExt; + use std::os::windows::io::AsRawHandle; + use std::time::{SystemTime, UNIX_EPOCH}; + + use super::*; + + #[test] + fn rejects_reparse_points_and_preserves_existing_rename_target() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = + std::env::temp_dir().join(format!("fs-safe-native-win-{}-{nonce}", std::process::id())); + fs::create_dir(&root).unwrap(); + fs::write(root.join("source"), b"source").unwrap(); + fs::write(root.join("target"), b"target").unwrap(); + let root_handle = OpenOptions::new() + .read(true) + .custom_flags(windows_sys::Win32::Storage::FileSystem::FILE_FLAG_BACKUP_SEMANTICS) + .open(&root) + .unwrap(); + let created = nt_open_relative( + root_handle.as_raw_handle() as HANDLE, + "created-dir", + FILE_GENERIC_READ | FILE_GENERIC_WRITE, + FILE_OPEN_IF, + FILE_DIRECTORY_FILE, + ) + .unwrap(); + drop(created); + assert!(root.join("created-dir").is_dir()); + // Windows Rust file handles are not CRT descriptors, so exercise the + // handle-relative primitive directly in this platform unit test. + let source = nt_open_relative( + root_handle.as_raw_handle() as HANDLE, + "source", + FILE_READ_ATTRIBUTES | DELETE_ACCESS, + FILE_OPEN, + FILE_NON_DIRECTORY_FILE, + ) + .unwrap(); + let error = set_rename_information( + source.0, + root_handle.as_raw_handle() as HANDLE, + "target", + "rename without replacement", + ) + .unwrap_err(); + assert_eq!(error.status, "EEXIST"); + assert_eq!(fs::read(root.join("target")).unwrap(), b"target"); + fs::remove_dir_all(root).unwrap(); + } +} diff --git a/native/src/windows_security.rs b/native/src/windows_security.rs new file mode 100644 index 0000000..770a85d --- /dev/null +++ b/native/src/windows_security.rs @@ -0,0 +1,545 @@ +use napi::{Env, Result}; +use napi_derive::napi; + +use crate::into_napi; +#[cfg(not(windows))] +use crate::native_error; + +#[napi(object)] +pub struct WindowsAceFlags { + pub raw: u32, + pub object_inherit: bool, + pub container_inherit: bool, + pub no_propagate_inherit: bool, + pub inherit_only: bool, + pub inherited: bool, + pub successful_access: bool, + pub failed_access: bool, +} + +#[napi(object)] +pub struct WindowsAccessControlEntry { + pub sid: String, + pub mask: u32, + pub ace_type: String, + pub flags: WindowsAceFlags, +} + +#[napi(object)] +pub struct WindowsSecurityFacts { + pub owner_sid: String, + pub current_user_sid: String, + pub owner_class: String, + pub world_writable: bool, + pub group_writable: bool, + pub world_readable: bool, + pub group_readable: bool, + pub fallback_required: bool, + pub dacl_present: bool, + pub is_local: bool, + pub ace_list_complete: bool, + pub unsupported_ace_types: Vec, + pub aces: Vec, +} + +#[cfg(any(windows, test))] +fn ace_flags(raw: u8) -> WindowsAceFlags { + WindowsAceFlags { + raw: raw.into(), + object_inherit: raw & 0x01 != 0, + container_inherit: raw & 0x02 != 0, + no_propagate_inherit: raw & 0x04 != 0, + inherit_only: raw & 0x08 != 0, + inherited: raw & 0x10 != 0, + successful_access: raw & 0x40 != 0, + failed_access: raw & 0x80 != 0, + } +} + +#[napi(js_name = "createPrivateDirectory")] +pub fn create_private_directory(env: Env, path: String) -> Result<()> { + #[cfg(windows)] + return into_napi(env, windows::create_private_directory(&path)); + #[cfg(not(windows))] + { + let _ = path; + into_napi( + env, + Err(native_error( + "ENOTSUP", + "private Windows directories are only available on Windows", + )), + ) + } +} + +#[napi(js_name = "readOwnerAndDacl")] +pub fn read_owner_and_dacl(env: Env, path: String) -> Result { + #[cfg(windows)] + return into_napi(env, windows::read_owner_and_dacl(&path)); + #[cfg(not(windows))] + { + let _ = path; + into_napi( + env, + Err(native_error( + "ENOTSUP", + "Windows owner and DACL inspection is only available on Windows", + )), + ) + } +} + +#[cfg(windows)] +mod windows { + use std::ffi::c_void; + use std::mem::zeroed; + use std::os::windows::ffi::OsStrExt; + use std::ptr::{null, null_mut}; + + use windows_sys::Win32::Foundation::{ + CloseHandle, ERROR_INSUFFICIENT_BUFFER, GetLastError, HANDLE, INVALID_HANDLE_VALUE, + LocalFree, + }; + use windows_sys::Win32::Security::Authorization::{ + ConvertSidToStringSidW, EXPLICIT_ACCESS_W, GRANT_ACCESS, GetSecurityInfo, SE_FILE_OBJECT, + SetEntriesInAclW, TRUSTEE_IS_SID, TRUSTEE_IS_UNKNOWN, + }; + use windows_sys::Win32::Security::{ + ACCESS_ALLOWED_ACE, ACCESS_DENIED_ACE, ACE_HEADER, ACL, CONTAINER_INHERIT_ACE, + CreateWellKnownSid, DACL_SECURITY_INFORMATION, EqualSid, GetAce, GetLengthSid, + GetTokenInformation, InitializeSecurityDescriptor, IsValidSid, IsWellKnownSid, + OBJECT_INHERIT_ACE, OWNER_SECURITY_INFORMATION, PSID, SE_DACL_PROTECTED, + SECURITY_ATTRIBUTES, SECURITY_DESCRIPTOR, SECURITY_MAX_SID_SIZE, + SetSecurityDescriptorControl, SetSecurityDescriptorDacl, SetSecurityDescriptorOwner, + TOKEN_QUERY, TOKEN_USER, TokenUser, WinAnonymousSid, WinAuthenticatedUserSid, + WinBuiltinAdministratorsSid, WinBuiltinGuestsSid, WinBuiltinUsersSid, WinInteractiveSid, + WinLocalSystemSid, WinNetworkSid, WinWorldSid, + }; + use windows_sys::Win32::Storage::FileSystem::{ + CreateDirectoryW, CreateFileW, FILE_ALL_ACCESS, FILE_FLAG_BACKUP_SEMANTICS, + FILE_READ_ATTRIBUTES, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, + GetFinalPathNameByHandleW, OPEN_EXISTING, + }; + use windows_sys::Win32::System::Threading::{GetCurrentProcess, OpenProcessToken}; + + use super::{WindowsAccessControlEntry, WindowsSecurityFacts, ace_flags}; + use crate::{NativeResult, native_error}; + + const GENERIC_READ: u32 = 0x8000_0000; + const GENERIC_WRITE: u32 = 0x4000_0000; + const GENERIC_ALL: u32 = 0x1000_0000; + const DELETE_ACCESS: u32 = 0x0001_0000; + const WRITE_DAC: u32 = 0x0004_0000; + const WRITE_OWNER: u32 = 0x0008_0000; + const READ_CONTROL: u32 = 0x0002_0000; + const FILE_READ_DATA: u32 = 0x0000_0001; + const FILE_WRITE_DATA: u32 = 0x0000_0002; + const FILE_APPEND_DATA: u32 = 0x0000_0004; + const FILE_READ_EA: u32 = 0x0000_0008; + const FILE_WRITE_EA: u32 = 0x0000_0010; + const FILE_DELETE_CHILD: u32 = 0x0000_0040; + const FILE_WRITE_ATTRIBUTES: u32 = 0x0000_0100; + const ACCESS_ALLOWED_ACE_TYPE: u8 = 0; + const ACCESS_DENIED_ACE_TYPE: u8 = 1; + const SECURITY_DESCRIPTOR_REVISION: u32 = 1; + + fn wide(value: &str) -> NativeResult> { + if value.encode_utf16().any(|unit| unit == 0) { + return Err(native_error("EINVAL", "Windows path contains a NUL byte")); + } + Ok(std::ffi::OsStr::new(value) + .encode_wide() + .chain(std::iter::once(0)) + .collect()) + } + + fn win_error(code: u32, operation: &str) -> napi::Error { + let typed = match code { + 5 => "EACCES", + 80 | 183 => "EEXIST", + 2 | 3 => "ENOENT", + _ => "EIO", + }; + native_error( + typed, + format!("{operation} failed with Windows error {code}"), + ) + } + + struct TokenSid { + _buffer: Vec, + sid: PSID, + } + + fn current_user_sid() -> NativeResult { + let mut token: HANDLE = null_mut(); + if unsafe { OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &mut token) } == 0 { + return Err(win_error(unsafe { GetLastError() }, "open process token")); + } + let result = (|| { + let mut needed = 0_u32; + unsafe { GetTokenInformation(token, TokenUser, null_mut(), 0, &mut needed) }; + if unsafe { GetLastError() } != ERROR_INSUFFICIENT_BUFFER || needed == 0 { + return Err(win_error(unsafe { GetLastError() }, "size token user")); + } + let mut buffer = vec![0_u8; needed as usize]; + if unsafe { + GetTokenInformation( + token, + TokenUser, + buffer.as_mut_ptr().cast(), + needed, + &mut needed, + ) + } == 0 + { + return Err(win_error(unsafe { GetLastError() }, "read token user")); + } + let sid = unsafe { (*(buffer.as_ptr().cast::())).User.Sid }; + Ok(TokenSid { + _buffer: buffer, + sid, + }) + })(); + unsafe { CloseHandle(token) }; + result + } + + fn well_known_sid(kind: i32) -> NativeResult> { + let mut buffer = vec![0_u8; SECURITY_MAX_SID_SIZE as usize]; + let mut size = buffer.len() as u32; + if unsafe { CreateWellKnownSid(kind, null_mut(), buffer.as_mut_ptr().cast(), &mut size) } + == 0 + { + return Err(win_error( + unsafe { GetLastError() }, + "create well-known SID", + )); + } + buffer.truncate(size as usize); + Ok(buffer) + } + + fn sid_string(sid: PSID) -> NativeResult { + let mut value = null_mut(); + if unsafe { ConvertSidToStringSidW(sid, &mut value) } == 0 { + return Err(win_error(unsafe { GetLastError() }, "format SID")); + } + let mut length = 0; + while unsafe { *value.add(length) } != 0 { + length += 1; + } + let result = String::from_utf16_lossy(unsafe { std::slice::from_raw_parts(value, length) }); + unsafe { LocalFree(value.cast()) }; + Ok(result.to_ascii_lowercase()) + } + + fn is_world_sid(sid: PSID) -> bool { + [ + WinWorldSid, + WinAuthenticatedUserSid, + WinBuiltinUsersSid, + WinAnonymousSid, + WinBuiltinGuestsSid, + WinInteractiveSid, + WinNetworkSid, + ] + .into_iter() + .any(|kind| unsafe { IsWellKnownSid(sid, kind) } != 0) + } + + fn can_read(mask: u32) -> bool { + mask & (GENERIC_ALL | GENERIC_READ | FILE_READ_DATA | FILE_READ_EA | FILE_READ_ATTRIBUTES) + != 0 + } + + fn can_write(mask: u32) -> bool { + mask & (GENERIC_ALL + | GENERIC_WRITE + | FILE_WRITE_DATA + | FILE_APPEND_DATA + | FILE_WRITE_EA + | FILE_WRITE_ATTRIBUTES + | FILE_DELETE_CHILD + | DELETE_ACCESS + | WRITE_DAC + | WRITE_OWNER) + != 0 + } + + fn parse_basic_ace( + raw: *mut c_void, + header: &ACE_HEADER, + ) -> NativeResult> { + let sid_offset = std::mem::offset_of!(ACCESS_ALLOWED_ACE, SidStart); + if (header.AceSize as usize) < sid_offset + 8 { + return Ok(None); + } + let (mask, sid, ace_type) = match header.AceType { + ACCESS_ALLOWED_ACE_TYPE => { + let ace = unsafe { &*(raw.cast::()) }; + ( + ace.Mask, + (&ace.SidStart as *const u32).cast_mut().cast::(), + "allow", + ) + } + ACCESS_DENIED_ACE_TYPE => { + let ace = unsafe { &*(raw.cast::()) }; + ( + ace.Mask, + (&ace.SidStart as *const u32).cast_mut().cast::(), + "deny", + ) + } + _ => return Ok(None), + }; + if unsafe { IsValidSid(sid) } == 0 { + return Ok(None); + } + let sid_length = unsafe { GetLengthSid(sid) } as usize; + if sid_length == 0 || sid_offset + sid_length > header.AceSize as usize { + return Ok(None); + } + Ok(Some(( + WindowsAccessControlEntry { + sid: sid_string(sid)?, + mask, + ace_type: ace_type.to_owned(), + flags: ace_flags(header.AceFlags), + }, + sid, + ))) + } + + fn open_security_handle(path: &[u16]) -> NativeResult { + let handle = unsafe { + CreateFileW( + path.as_ptr(), + FILE_READ_ATTRIBUTES | READ_CONTROL, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + null(), + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + return Err(win_error( + unsafe { GetLastError() }, + "open path for locality check", + )); + } + Ok(handle) + } + + fn is_local_handle(handle: HANDLE) -> NativeResult { + let needed = unsafe { GetFinalPathNameByHandleW(handle, null_mut(), 0, 0) }; + if needed == 0 { + return Err(win_error(unsafe { GetLastError() }, "size final path")); + } + let mut buffer = vec![0_u16; needed as usize + 1]; + let written = unsafe { + GetFinalPathNameByHandleW(handle, buffer.as_mut_ptr(), buffer.len() as u32, 0) + }; + if written == 0 || written as usize >= buffer.len() { + return Err(win_error(unsafe { GetLastError() }, "resolve final path")); + } + let final_path = String::from_utf16_lossy(&buffer[..written as usize]); + Ok(!final_path.starts_with(r"\\?\UNC\") + && (!final_path.starts_with(r"\\") || final_path.starts_with(r"\\?\"))) + } + + pub fn read_owner_and_dacl(path: &str) -> NativeResult { + let path = wide(path)?; + let current = current_user_sid()?; + let handle = open_security_handle(&path)?; + let local = is_local_handle(handle).unwrap_or(false); + let mut owner = null_mut(); + let mut dacl: *mut ACL = null_mut(); + let mut descriptor = null_mut(); + let status = unsafe { + GetSecurityInfo( + handle, + SE_FILE_OBJECT, + OWNER_SECURITY_INFORMATION | DACL_SECURITY_INFORMATION, + &mut owner, + null_mut(), + &mut dacl, + null_mut(), + &mut descriptor, + ) + }; + if status != 0 { + unsafe { CloseHandle(handle) }; + return Err(win_error(status, "read owner and DACL")); + } + let result = (|| { + let owner_class = if unsafe { EqualSid(owner, current.sid) } != 0 { + "current-user" + } else if unsafe { IsWellKnownSid(owner, WinLocalSystemSid) } != 0 { + "system" + } else if unsafe { IsWellKnownSid(owner, WinBuiltinAdministratorsSid) } != 0 { + "administrators" + } else { + "foreign" + }; + let mut facts = WindowsSecurityFacts { + owner_sid: sid_string(owner)?, + current_user_sid: sid_string(current.sid)?, + owner_class: owner_class.to_owned(), + world_writable: dacl.is_null(), + group_writable: false, + world_readable: dacl.is_null(), + group_readable: false, + fallback_required: !local, + dacl_present: !dacl.is_null(), + is_local: local, + ace_list_complete: true, + unsupported_ace_types: Vec::new(), + aces: Vec::new(), + }; + if !dacl.is_null() { + let count = unsafe { (*dacl).AceCount } as u32; + for index in 0..count { + let mut raw = null_mut(); + if unsafe { GetAce(dacl, index, &mut raw) } == 0 || raw.is_null() { + facts.fallback_required = true; + facts.ace_list_complete = false; + continue; + } + let header = unsafe { &*(raw.cast::()) }; + let Some((entry, sid)) = parse_basic_ace(raw, header)? else { + facts.fallback_required = true; + facts.ace_list_complete = false; + facts.unsupported_ace_types.push(header.AceType.into()); + continue; + }; + let mask = entry.mask; + let inherit_only = entry.flags.inherit_only; + facts.aces.push(entry); + if inherit_only || header.AceType == ACCESS_DENIED_ACE_TYPE { + continue; + } + let trusted = unsafe { EqualSid(sid, current.sid) } != 0 + || unsafe { IsWellKnownSid(sid, WinLocalSystemSid) } != 0 + || unsafe { IsWellKnownSid(sid, WinBuiltinAdministratorsSid) } != 0; + if trusted { + continue; + } + if is_world_sid(sid) { + facts.world_readable |= can_read(mask); + facts.world_writable |= can_write(mask); + } else { + facts.group_readable |= can_read(mask); + facts.group_writable |= can_write(mask); + } + } + } + Ok(facts) + })(); + unsafe { LocalFree(descriptor) }; + unsafe { CloseHandle(handle) }; + result + } + + pub fn create_private_directory(path: &str) -> NativeResult<()> { + let path_wide = wide(path)?; + let current = current_user_sid()?; + let system = well_known_sid(WinLocalSystemSid)?; + let administrators = well_known_sid(WinBuiltinAdministratorsSid)?; + let sids: [PSID; 3] = [ + current.sid, + system.as_ptr().cast_mut().cast(), + administrators.as_ptr().cast_mut().cast(), + ]; + let mut entries: [EXPLICIT_ACCESS_W; 3] = unsafe { zeroed() }; + for (entry, sid) in entries.iter_mut().zip(sids) { + entry.grfAccessPermissions = FILE_ALL_ACCESS; + entry.grfAccessMode = GRANT_ACCESS; + entry.grfInheritance = OBJECT_INHERIT_ACE | CONTAINER_INHERIT_ACE; + entry.Trustee.TrusteeForm = TRUSTEE_IS_SID; + entry.Trustee.TrusteeType = TRUSTEE_IS_UNKNOWN; + entry.Trustee.ptstrName = sid.cast(); + } + let mut acl: *mut ACL = null_mut(); + let status = + unsafe { SetEntriesInAclW(entries.len() as u32, entries.as_ptr(), null(), &mut acl) }; + if status != 0 { + return Err(win_error(status, "build private directory DACL")); + } + let mut created = false; + let result = (|| { + let mut descriptor: SECURITY_DESCRIPTOR = unsafe { zeroed() }; + let descriptor_ptr = (&mut descriptor as *mut SECURITY_DESCRIPTOR).cast(); + if unsafe { InitializeSecurityDescriptor(descriptor_ptr, SECURITY_DESCRIPTOR_REVISION) } + == 0 + || unsafe { SetSecurityDescriptorOwner(descriptor_ptr, current.sid, 0) } == 0 + || unsafe { SetSecurityDescriptorDacl(descriptor_ptr, 1, acl, 0) } == 0 + || unsafe { + SetSecurityDescriptorControl( + descriptor_ptr, + SE_DACL_PROTECTED, + SE_DACL_PROTECTED, + ) + } == 0 + { + return Err(win_error( + unsafe { GetLastError() }, + "build private directory security descriptor", + )); + } + let attributes = SECURITY_ATTRIBUTES { + nLength: std::mem::size_of::() as u32, + lpSecurityDescriptor: descriptor_ptr, + bInheritHandle: 0, + }; + if unsafe { CreateDirectoryW(path_wide.as_ptr(), &attributes) } == 0 { + return Err(win_error( + unsafe { GetLastError() }, + "create private directory", + )); + } + created = true; + let facts = read_owner_and_dacl(path)?; + if facts.owner_class != "current-user" + || facts.world_readable + || facts.world_writable + || facts.group_readable + || facts.group_writable + || facts.fallback_required + { + return Err(native_error( + "EACCES", + "filesystem did not enforce the private directory DACL", + )); + } + Ok(()) + })(); + unsafe { LocalFree(acl.cast()) }; + if created && result.is_err() { + unsafe { + windows_sys::Win32::Storage::FileSystem::RemoveDirectoryW(path_wide.as_ptr()) + }; + } + result + } +} + +#[cfg(test)] +mod tests { + use super::ace_flags; + + #[test] + fn decodes_ace_inheritance_and_audit_flags() { + let flags = ace_flags(0x01 | 0x02 | 0x04 | 0x08 | 0x10 | 0x40 | 0x80); + assert!(flags.object_inherit); + assert!(flags.container_inherit); + assert!(flags.no_propagate_inherit); + assert!(flags.inherit_only); + assert!(flags.inherited); + assert!(flags.successful_access); + assert!(flags.failed_access); + } +} diff --git a/npm/darwin-arm64/package.json b/npm/darwin-arm64/package.json new file mode 100644 index 0000000..fe50c6b --- /dev/null +++ b/npm/darwin-arm64/package.json @@ -0,0 +1,14 @@ +{ + "name": "@openclaw/fs-safe-native-darwin-arm64", + "version": "0.5.0-dev.0", + "description": "Native binding for @openclaw/fs-safe on macOS arm64.", + "license": "MIT", + "author": "OpenClaw Team ", + "repository": "github:openclaw/fs-safe", + "main": "fs-safe-native.darwin-arm64.node", + "files": ["fs-safe-native.darwin-arm64.node"], + "os": ["darwin"], + "cpu": ["arm64"], + "engines": { "node": ">=22" }, + "publishConfig": { "access": "public", "provenance": true } +} diff --git a/npm/darwin-x64/package.json b/npm/darwin-x64/package.json new file mode 100644 index 0000000..2f835a3 --- /dev/null +++ b/npm/darwin-x64/package.json @@ -0,0 +1,14 @@ +{ + "name": "@openclaw/fs-safe-native-darwin-x64", + "version": "0.5.0-dev.0", + "description": "Native binding for @openclaw/fs-safe on macOS x64.", + "license": "MIT", + "author": "OpenClaw Team ", + "repository": "github:openclaw/fs-safe", + "main": "fs-safe-native.darwin-x64.node", + "files": ["fs-safe-native.darwin-x64.node"], + "os": ["darwin"], + "cpu": ["x64"], + "engines": { "node": ">=22" }, + "publishConfig": { "access": "public", "provenance": true } +} diff --git a/npm/linux-arm64-gnu/package.json b/npm/linux-arm64-gnu/package.json new file mode 100644 index 0000000..44ac11d --- /dev/null +++ b/npm/linux-arm64-gnu/package.json @@ -0,0 +1,15 @@ +{ + "name": "@openclaw/fs-safe-native-linux-arm64-gnu", + "version": "0.5.0-dev.0", + "description": "Native binding for @openclaw/fs-safe on Linux arm64 glibc.", + "license": "MIT", + "author": "OpenClaw Team ", + "repository": "github:openclaw/fs-safe", + "main": "fs-safe-native.linux-arm64-gnu.node", + "files": ["fs-safe-native.linux-arm64-gnu.node"], + "os": ["linux"], + "cpu": ["arm64"], + "libc": ["glibc"], + "engines": { "node": ">=22" }, + "publishConfig": { "access": "public", "provenance": true } +} diff --git a/npm/linux-arm64-musl/package.json b/npm/linux-arm64-musl/package.json new file mode 100644 index 0000000..1124f89 --- /dev/null +++ b/npm/linux-arm64-musl/package.json @@ -0,0 +1,15 @@ +{ + "name": "@openclaw/fs-safe-native-linux-arm64-musl", + "version": "0.5.0-dev.0", + "description": "Native binding for @openclaw/fs-safe on Linux arm64 musl.", + "license": "MIT", + "author": "OpenClaw Team ", + "repository": "github:openclaw/fs-safe", + "main": "fs-safe-native.linux-arm64-musl.node", + "files": ["fs-safe-native.linux-arm64-musl.node"], + "os": ["linux"], + "cpu": ["arm64"], + "libc": ["musl"], + "engines": { "node": ">=22" }, + "publishConfig": { "access": "public", "provenance": true } +} diff --git a/npm/linux-x64-gnu/package.json b/npm/linux-x64-gnu/package.json new file mode 100644 index 0000000..9fd1c9b --- /dev/null +++ b/npm/linux-x64-gnu/package.json @@ -0,0 +1,15 @@ +{ + "name": "@openclaw/fs-safe-native-linux-x64-gnu", + "version": "0.5.0-dev.0", + "description": "Native binding for @openclaw/fs-safe on Linux x64 glibc.", + "license": "MIT", + "author": "OpenClaw Team ", + "repository": "github:openclaw/fs-safe", + "main": "fs-safe-native.linux-x64-gnu.node", + "files": ["fs-safe-native.linux-x64-gnu.node"], + "os": ["linux"], + "cpu": ["x64"], + "libc": ["glibc"], + "engines": { "node": ">=22" }, + "publishConfig": { "access": "public", "provenance": true } +} diff --git a/npm/linux-x64-musl/package.json b/npm/linux-x64-musl/package.json new file mode 100644 index 0000000..58b8286 --- /dev/null +++ b/npm/linux-x64-musl/package.json @@ -0,0 +1,15 @@ +{ + "name": "@openclaw/fs-safe-native-linux-x64-musl", + "version": "0.5.0-dev.0", + "description": "Native binding for @openclaw/fs-safe on Linux x64 musl.", + "license": "MIT", + "author": "OpenClaw Team ", + "repository": "github:openclaw/fs-safe", + "main": "fs-safe-native.linux-x64-musl.node", + "files": ["fs-safe-native.linux-x64-musl.node"], + "os": ["linux"], + "cpu": ["x64"], + "libc": ["musl"], + "engines": { "node": ">=22" }, + "publishConfig": { "access": "public", "provenance": true } +} diff --git a/npm/win32-x64-msvc/package.json b/npm/win32-x64-msvc/package.json new file mode 100644 index 0000000..d4a6ea5 --- /dev/null +++ b/npm/win32-x64-msvc/package.json @@ -0,0 +1,14 @@ +{ + "name": "@openclaw/fs-safe-native-win32-x64-msvc", + "version": "0.5.0-dev.0", + "description": "Native binding for @openclaw/fs-safe on Windows x64.", + "license": "MIT", + "author": "OpenClaw Team ", + "repository": "github:openclaw/fs-safe", + "main": "fs-safe-native.win32-x64-msvc.node", + "files": ["fs-safe-native.win32-x64-msvc.node"], + "os": ["win32"], + "cpu": ["x64"], + "engines": { "node": ">=22" }, + "publishConfig": { "access": "public", "provenance": true } +} diff --git a/package.json b/package.json index 18878b1..4aaf785 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@openclaw/fs-safe", - "version": "0.4.7", + "version": "0.5.0-dev.0", "description": "Capability-style filesystem roots for Node.js apps that handle untrusted relative paths.", "keywords": [ "filesystem", @@ -120,6 +120,7 @@ }, "scripts": { "benchmark": "node scripts/benchmark.mjs", + "benchmark:publish": "pnpm build && node scripts/bench-publish.mjs", "build": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\" && tsc -p tsconfig.json", "lint:file-size": "node scripts/check-file-size.mjs", "lint:fs-boundary": "node scripts/check-fs-boundary-primitives.mjs", @@ -129,7 +130,10 @@ "test:security": "vitest run test/fs-safe.test.ts test/read-boundary-bypass.test.ts test/write-boundary-bypass.test.ts test/additional-boundary-bypass.test.ts test/adversarial-boundary-payloads.test.ts", "check": "pnpm lint:file-size && pnpm lint:fs-boundary && pnpm build && pnpm test && node scripts/check-pack.mjs", "docs:site": "node scripts/build-docs-site.mjs", + "native:build": "pnpm --filter @openclaw/fs-safe-native build", + "native:test": "cargo test --manifest-path native/Cargo.toml", "pack:check": "pnpm build && node scripts/check-pack.mjs", + "release-graph:smoke": "node scripts/release-graph-smoke.mjs", "check:changed": "pnpm run check", "release:notes": "node scripts/release-notes.mjs", "test:changed": "pnpm run test", @@ -139,10 +143,12 @@ "crabbox:warmup": "crabbox warmup" }, "optionalDependencies": { + "@openclaw/fs-safe-native": "0.5.0-dev.0", "jszip": "^3.10.1", "tar": "7.5.21" }, "devDependencies": { + "@napi-rs/cli": "3.7.4", "@types/node": "^26.1.1", "@vitest/coverage-v8": "4.1.10", "typescript": "^7.0.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index be54e85..291bbc5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: devDependencies: + '@napi-rs/cli': + specifier: 3.7.4 + version: 3.7.4(@emnapi/core@1.11.2)(@emnapi/runtime@1.11.2)(@types/node@26.1.1) '@types/node': specifier: ^26.1.1 version: 26.1.1 @@ -24,6 +27,9 @@ importers: specifier: ^4.1.10 version: 4.1.10(@types/node@26.1.1)(@vitest/coverage-v8@4.1.10)(vite@8.1.5(@types/node@26.1.1)(esbuild@0.28.1)) optionalDependencies: + '@openclaw/fs-safe-native': + specifier: 0.5.0-dev.0 + version: link:native jszip: specifier: ^3.10.1 version: 3.10.1 @@ -31,6 +37,44 @@ importers: specifier: 7.5.21 version: 7.5.21 + native: + optionalDependencies: + '@openclaw/fs-safe-native-darwin-arm64': + specifier: 0.5.0-dev.0 + version: link:../npm/darwin-arm64 + '@openclaw/fs-safe-native-darwin-x64': + specifier: 0.5.0-dev.0 + version: link:../npm/darwin-x64 + '@openclaw/fs-safe-native-linux-arm64-gnu': + specifier: 0.5.0-dev.0 + version: link:../npm/linux-arm64-gnu + '@openclaw/fs-safe-native-linux-arm64-musl': + specifier: 0.5.0-dev.0 + version: link:../npm/linux-arm64-musl + '@openclaw/fs-safe-native-linux-x64-gnu': + specifier: 0.5.0-dev.0 + version: link:../npm/linux-x64-gnu + '@openclaw/fs-safe-native-linux-x64-musl': + specifier: 0.5.0-dev.0 + version: link:../npm/linux-x64-musl + '@openclaw/fs-safe-native-win32-x64-msvc': + specifier: 0.5.0-dev.0 + version: link:../npm/win32-x64-msvc + + npm/darwin-arm64: {} + + npm/darwin-x64: {} + + npm/linux-arm64-gnu: {} + + npm/linux-arm64-musl: {} + + npm/linux-x64-gnu: {} + + npm/linux-x64-musl: {} + + npm/win32-x64-msvc: {} + packages: '@babel/helper-string-parser@7.29.7': @@ -57,9 +101,15 @@ packages: '@emnapi/core@1.11.1': resolution: {integrity: sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==} + '@emnapi/core@1.11.2': + resolution: {integrity: sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA==} + '@emnapi/runtime@1.11.1': resolution: {integrity: sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==} + '@emnapi/runtime@1.11.2': + resolution: {integrity: sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==} + '@emnapi/wasi-threads@1.2.2': resolution: {integrity: sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==} @@ -219,6 +269,140 @@ packages: cpu: [x64] os: [win32] + '@inquirer/ansi@2.0.7': + resolution: {integrity: sha512-3eTuUO1vH2cZm2ZKHeQxnOqlTi9EfZDGgIe3BL3I4u+rJHocr9Fz86M4fjYABPvFnQG/gGK551HqDiIcETwU6Q==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + + '@inquirer/checkbox@5.2.1': + resolution: {integrity: sha512-b6xmA/VlTe0ZgDQHDui+Nav470u7u49nRd8/iuhOcQPO9Ch7lGuogydhi2VOmNlZ+zXcM8IcPuNSwQcdJaF/kw==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/confirm@6.1.1': + resolution: {integrity: sha512-eb8DBZcz/2qHWQda4rk2JiQk5h9QV/cVHi1yjt0f69WFZMRFn0sJTye3EAP8icut8UDMjQPsaH5KbcOogefrFQ==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/core@11.2.1': + resolution: {integrity: sha512-Qd6GJT1yVyrZZCfN8W2qKF5ApmqryXRhRKCuip8h01x2w/esJQ2XIYc6f9abMIHgKQdBfFTSOdbHRLAhuM09UA==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/editor@5.2.2': + resolution: {integrity: sha512-ZRVd/oD+sYsUd5zVm0NflqEzlqfYCyHNsqkHl2oWXEUHs12tCbcSFi+wVFEvD8+LGRaMUsVrE7qeo6lSG/S1Vg==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/expand@5.1.1': + resolution: {integrity: sha512-YmQpenjbFSHAK3sOd44puHh3V1KXXr+JiNpUztoSQ4drLh2rTVzTap/YtlAVu/5xavifIlBfNEzJ/neZJ1a/1g==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/external-editor@3.0.3': + resolution: {integrity: sha512-6thf5I8q7lZwzGLAxPaaGEREEkZ3nyePPDQ1oyobblxmEE8mqTLguScP7pDjUTAibiyb4hfXl+qjUEJ+di/aNA==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/figures@2.0.7': + resolution: {integrity: sha512-aJ8TBPOGB6f/2qziPfElISTCEd5XOYTFckA2SGjhNmiKzfK/u4ot3v0DUzGVdUnKjN10EqnnEPck36BkyfLnJw==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + + '@inquirer/input@5.1.2': + resolution: {integrity: sha512-9K/DDBSQpOyZSkt6sOVP9Vo0TR7atX2kuILsUu0x3wVcVbe97lJwIJKMLdMw25tDYuXl/qp6erT0Xs1rfmcfZg==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/number@4.1.1': + resolution: {integrity: sha512-XF4IXAbPnGPgw0wsbC/i2tPcyfdZgDpUlhsqU0SfT4IRIGWha6Xm9VRgN5yYxJq+jnyXlfXI/nQ3ulfk0iEICA==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/password@5.1.1': + resolution: {integrity: sha512-3XBfF7DAsp5qeDsvN5Rd1HmbNokVvEQoUM0QLrRcybC9nX96w3Pbmu7qUsb3IT3J3jBvs2+mTXaKHOUsgHMLzg==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/prompts@8.5.2': + resolution: {integrity: sha512-IYR/3C/paEVVQYQvdDlFZVjRCJVYHHON0XXMH91KO9GSxs0TdKYWlUdvfQl2EfAHDxUaN3IBffkE/BDTh5nJ6g==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/rawlist@5.3.1': + resolution: {integrity: sha512-QqdTqQddL3qPX/PPrjobpsO25NZ4dWXgTLenrR445L2ptLEYE6Z+PD5c5CNDJNx4ugRgELAIpSIJxZaO2jJ2Og==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/search@4.2.1': + resolution: {integrity: sha512-xJj8QWKRSrfKoBIITLZK61dD3zwo0Rz11fgDImku30/Oe81zMdIdGgrLY2h6RkJ+KZ/GhNYIRMKnH/62qBTA5g==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/select@5.2.1': + resolution: {integrity: sha512-FlDndEUww8m7BfukO2nJa25vhD+H5jxxCv4oGioKqzyWz3nPHhhw4LKdYRSlXuAx7DsdWia7iyaBPKKS95Evfw==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + + '@inquirer/type@4.0.7': + resolution: {integrity: sha512-t28inv14nMQ1PhKpsJPY+kEs/c00qzeCOS2gTNRyTjG5d6qsVA2fItxW4hkvGZ5lvanGLdtCzVIx5dwdRpN1+g==} + engines: {node: '>=23.5.0 || ^22.13.0 || ^20.17.0'} + peerDependencies: + '@types/node': '>=18' + peerDependenciesMeta: + '@types/node': + optional: true + '@isaacs/fs-minipass@4.0.1': resolution: {integrity: sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==} engines: {node: '>=18.0.0'} @@ -233,12 +417,411 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@napi-rs/cli@3.7.4': + resolution: {integrity: sha512-idELIUceJ9zPgx/shcMt1fSSsteFG68v56RIXi4qUm7OYuJOt7CoMj2KnHVmbOqXgUHWQU1lCULrtQdZDG4F5A==} + engines: {node: '>= 16'} + hasBin: true + peerDependencies: + '@emnapi/runtime': ^1.7.1 + peerDependenciesMeta: + '@emnapi/runtime': + optional: true + + '@napi-rs/cross-toolchain@1.0.3': + resolution: {integrity: sha512-ENPfLe4937bsKVTDA6zdABx4pq9w0tHqRrJHyaGxgaPq03a2Bd1unD5XSKjXJjebsABJ+MjAv1A2OvCgK9yehg==} + peerDependencies: + '@napi-rs/cross-toolchain-arm64-target-aarch64': ^1.0.3 + '@napi-rs/cross-toolchain-arm64-target-armv7': ^1.0.3 + '@napi-rs/cross-toolchain-arm64-target-ppc64le': ^1.0.3 + '@napi-rs/cross-toolchain-arm64-target-s390x': ^1.0.3 + '@napi-rs/cross-toolchain-arm64-target-x86_64': ^1.0.3 + '@napi-rs/cross-toolchain-x64-target-aarch64': ^1.0.3 + '@napi-rs/cross-toolchain-x64-target-armv7': ^1.0.3 + '@napi-rs/cross-toolchain-x64-target-ppc64le': ^1.0.3 + '@napi-rs/cross-toolchain-x64-target-s390x': ^1.0.3 + '@napi-rs/cross-toolchain-x64-target-x86_64': ^1.0.3 + peerDependenciesMeta: + '@napi-rs/cross-toolchain-arm64-target-aarch64': + optional: true + '@napi-rs/cross-toolchain-arm64-target-armv7': + optional: true + '@napi-rs/cross-toolchain-arm64-target-ppc64le': + optional: true + '@napi-rs/cross-toolchain-arm64-target-s390x': + optional: true + '@napi-rs/cross-toolchain-arm64-target-x86_64': + optional: true + '@napi-rs/cross-toolchain-x64-target-aarch64': + optional: true + '@napi-rs/cross-toolchain-x64-target-armv7': + optional: true + '@napi-rs/cross-toolchain-x64-target-ppc64le': + optional: true + '@napi-rs/cross-toolchain-x64-target-s390x': + optional: true + '@napi-rs/cross-toolchain-x64-target-x86_64': + optional: true + + '@napi-rs/lzma-android-arm-eabi@1.5.1': + resolution: {integrity: sha512-sahBe4ko2Z69NPTddaX6ZgbQZu9SDoITxw1S3dWl1gAGynZG34qHHCT8UaUMFxf3h3zMhCJjEzz4basaBxiTuQ==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [arm] + os: [android] + + '@napi-rs/lzma-android-arm64@1.5.1': + resolution: {integrity: sha512-7tkQAJJuBHxAxiEBNFgSTpvrtGpbwZYYJUSOmGEK3OfbdbNeoT2rdBxpM/gY1s+itEVbtOSlpaRPPG19MnwOzA==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [arm64] + os: [android] + + '@napi-rs/lzma-darwin-arm64@1.5.1': + resolution: {integrity: sha512-XWX8gtF+GHGk3nH3Wm3QUZNcxw9QHsFVZz3MzVLhWWHhceede1J4/vD+3dj3E1iKB9G6mualaZxOoD08R3E+7g==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [arm64] + os: [darwin] + + '@napi-rs/lzma-darwin-x64@1.5.1': + resolution: {integrity: sha512-CfsqUpMTI1z8enrA/b+GcHM6YDI8D0kqCiqPYEnst4rbOABQ9KZ92ybTTNnlnZ7A017WoMZKUEWc36KXDwi0xg==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [x64] + os: [darwin] + + '@napi-rs/lzma-freebsd-x64@1.5.1': + resolution: {integrity: sha512-bTyNfg90FXIgE61U7l14aMmVOqRQ6AyP5JMT3jmCStaZI18apLNPdzZ8i7yqxZfKvRMVfPjE2brXIw27c+RRgA==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [x64] + os: [freebsd] + + '@napi-rs/lzma-linux-arm-gnueabihf@1.5.1': + resolution: {integrity: sha512-vNE+D8nrw+eOkBsdKCsmDhowDV3pIMKXEhedvXfbgrWbrO7GlZJH+RXL+X+RYLxGwi8Ym61ZMt15sIOnNmh9Sw==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [arm] + os: [linux] + + '@napi-rs/lzma-linux-arm64-gnu@1.5.1': + resolution: {integrity: sha512-csUem4WgoKGTprv/pOPm9UIWbb+hrfUwYXefpTHPAEGVFLl5behEFabisJ7FtihCa3yG2Efcl+yw25rlhhrIYw==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@napi-rs/lzma-linux-arm64-musl@1.5.1': + resolution: {integrity: sha512-kB/xhlVN1eLvVmDJSKZEjp5Gg2xDYexNrB5jwpSMbOkeGS6N9AasByPBg5VqCpMYC+zZi7DM458DRhtWYhqXTQ==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@napi-rs/lzma-linux-ppc64-gnu@1.5.1': + resolution: {integrity: sha512-s28RW0W1yBWQc1nbPdF7tp14koqslY3ZWLVI8uaanX292Dc6ezd4NPVwxEoCNBVON/oD7BmUbWGtyFvmm7dQ5A==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@napi-rs/lzma-linux-riscv64-gnu@1.5.1': + resolution: {integrity: sha512-+lGNwYlIN14YPMTNvYtIJJqHFevDTd6Juw/1NmXbWx/iRd/LLrjhlM/yluMX6pxs6NkOGsuuEXJJrbbEUS59OQ==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@napi-rs/lzma-linux-s390x-gnu@1.5.1': + resolution: {integrity: sha512-PB44FFWWFrLeQowhcep1hPD1YcLqKlnnY60RMU74qrxTlr4YGEyzeMItJqh2uivBfv9kQScOF/B0J9+Vab/oyw==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@napi-rs/lzma-linux-x64-gnu@1.5.1': + resolution: {integrity: sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@napi-rs/lzma-linux-x64-musl@1.5.1': + resolution: {integrity: sha512-I3nsYrWtrW9JpeCr+mkJIVDt0HY3m6qVUBs5vTtoIvJQxwqf1PBXSy5IS7T53ksQFH2kd2UX8rLxJ7B4WISpZg==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [x64] + os: [linux] + libc: [musl] + + '@napi-rs/lzma-wasm32-wasi@1.5.1': + resolution: {integrity: sha512-gy3wwPBa6+XEyA4fUzq6CClrXA1ajXjuVf5zbnHytJRgoHznj+mvpU3+co2fxXwqTCmIpn6KrzqH5bRDztBPhA==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [wasm32] + + '@napi-rs/lzma-win32-arm64-msvc@1.5.1': + resolution: {integrity: sha512-dK+huOsHiyH6oJjij+cnjqFCakk2HgWmpI12Xm4pLUyPphe4ebYoJBgehaNAxprmjFqBQ7nL95YPVz9BHyqmPg==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [arm64] + os: [win32] + + '@napi-rs/lzma-win32-ia32-msvc@1.5.1': + resolution: {integrity: sha512-dGE8L+0EQ+GyU9ap9InqB/t/PmPG/bLj918q7OsJ29FuTdn8fK4OX3U4IQZhylHIA+/dQ/SXJk5n4yfah2XVvA==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [ia32] + os: [win32] + + '@napi-rs/lzma-win32-x64-msvc@1.5.1': + resolution: {integrity: sha512-EKW4t/iqdCT/xnd5t9oXLvVER/PMNAWXKqUAl3fgvUcOILeZIIht77/dVnfFcc9htA/DCBXC/6YQWdW+LusjFA==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [x64] + os: [win32] + + '@napi-rs/lzma@1.5.1': + resolution: {integrity: sha512-sgOZ89+y8cDbY+3WbzR8CtIhCuFRWotZ9/2PjPVDJHz6np5KFTAev0DrwiyTJTgFsCRDhfGlbmhMgyhHbWdZ6g==} + engines: {node: ^22.20 || ^24.12 || >=25} + + '@napi-rs/tar-android-arm-eabi@1.1.1': + resolution: {integrity: sha512-cAhnA10cSusAUbcE9HtjQY/tZ9BH/0w2sKtRcQc94TzIlnm7QSr1htJSd/PPrbWNPtrv1orXb2CkrHlVlbnlHA==} + engines: {node: '>= 10'} + cpu: [arm] + os: [android] + + '@napi-rs/tar-android-arm64@1.1.1': + resolution: {integrity: sha512-EslUWHCDBY/g5abTPBiHLsMaML4GagV0TXLm5WL9hAjx/DDtlxz9fegMb77RJ+f7nFLOIsUxF/3QWFvgOT0sMQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@napi-rs/tar-darwin-arm64@1.1.1': + resolution: {integrity: sha512-+A42/6ES5G9CQ35BOwzwA+WBjLID28r2jNPgc0dteD2hhClIhng0mva7D2ujUlXBNmgNOsr1LHn3stA4uTf4NQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@napi-rs/tar-darwin-x64@1.1.1': + resolution: {integrity: sha512-RYtE8w1dkEvj8hSJCDV5Jw0Rz2i13fsM7u893zv5O9n/4Ad5GNsw/f4RQ7/0YGSFaenkVxqPFrjmEvUHlKzsrg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@napi-rs/tar-freebsd-x64@1.1.1': + resolution: {integrity: sha512-rEepBvCJUwcuvUYkY83e8aot8RsR5Jcnal4PsG3tbWGKW1yAvcXhyMXf0fN6ZGpVRZFnB+FJqDyBxvsCPEXKhw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [freebsd] + + '@napi-rs/tar-linux-arm-gnueabihf@1.1.1': + resolution: {integrity: sha512-an1bJdfyhI5FpZYyTQ20mrqwR+a676i8GkaYc4Uy12dH/a7TJIfrK6Qa2Gm46arZvxUvx56qxoRKXbpOjUPvwA==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@napi-rs/tar-linux-arm64-gnu@1.1.1': + resolution: {integrity: sha512-w++Vtx36T2yHTKws7GVnmHHcUT1ybB59xLWSh9A8bwEpJVG4dG7Qub9mFe5cpcbfrJ+XP2mKKxC3oUJSunK3iQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@napi-rs/tar-linux-arm64-musl@1.1.1': + resolution: {integrity: sha512-Rh6UFhNtj3i4deJHOBINFIeRL0072mgbeyuK5rl1HokKnNoMKx8qKIZNEzBTTqpogMfDHWGvzyTQdnVxes5dpA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@napi-rs/tar-linux-ppc64-gnu@1.1.1': + resolution: {integrity: sha512-Cp+AxFbv9zcyAXtnzQi0OzmgDnQgy2w9D4Ubr+iwzMtVgJcztzcEoCcCrN1k2ATdEB01LX2Vb49IaocGOZhC9Q==} + engines: {node: '>= 10'} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@napi-rs/tar-linux-s390x-gnu@1.1.1': + resolution: {integrity: sha512-ZyscC3SYKTBWyDRYjLOKAd5TyJ7q0KACRdQ8bWrb3rgrra1CCIJD66CsGTH6Dh0AVSdfLwZ8MfIIXU6+14BMjQ==} + engines: {node: '>= 10'} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@napi-rs/tar-linux-x64-gnu@1.1.1': + resolution: {integrity: sha512-LlIv+zg4fiOQge9LQX/ieBdRWE2fhVDjCTHxnunZkbugNmdhdelxWf1RpZb/6ZujWpNF4LPu4N/MW7ygg2oYAQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@napi-rs/tar-linux-x64-musl@1.1.1': + resolution: {integrity: sha512-gZBeoKLjanOVj55qk4EMu13P2i9M0SuINmlGQkOxm1niIJofexzddHUYtqO5o/5QqtyL8lADmAcZplLILMLhHA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@napi-rs/tar-wasm32-wasi@1.1.1': + resolution: {integrity: sha512-rwtQ1Mdt/ft6g6I54fJzbUeLspl4yTwj6I3UJ6mitKnrN42soJkcDrdh3Y/FGvlpqZTad2YMQ96fGJl3EtAm2Q==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + + '@napi-rs/tar-win32-arm64-msvc@1.1.1': + resolution: {integrity: sha512-30PVp1AehRpfwxmv5wI4cg0yj3WmWBsZ+1QnLGnvEELu7Eu/+dhNU0nrmhI7VfPgLwSRK2eg9DQTB3tP7Wv9bA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@napi-rs/tar-win32-ia32-msvc@1.1.1': + resolution: {integrity: sha512-aI3/rmz+izUChiSeaPxcasAOxhf3FpJNuIHMXlxS/vpW+HIxUsSDR5+XV61PEG5DL4L/75iENVUxmSGM5l2yaw==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + + '@napi-rs/tar-win32-x64-msvc@1.1.1': + resolution: {integrity: sha512-yJsB2IsrODQVLKbm2Fg1nHiVRbEj49mSPbj4x7JPZWJI0jGVPjohE2Sif0FBbx8OxsVoUODvS0BwksZZ8jl/OA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@napi-rs/tar@1.1.1': + resolution: {integrity: sha512-p6q2HhUc5vwH1CNwfOcrhLoxfgn8ust8Sqlfx+sA4VzAcp1cMbvbkl99tZZlDqOjCHgQNSiTfk/yWPjl/D42qA==} + engines: {node: '>= 10'} + '@napi-rs/wasm-runtime@1.1.6': resolution: {integrity: sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==} peerDependencies: '@emnapi/core': ^1.7.1 '@emnapi/runtime': ^1.7.1 + '@napi-rs/wasm-tools-android-arm-eabi@1.0.1': + resolution: {integrity: sha512-lr07E/l571Gft5v4aA1dI8koJEmF1F0UigBbsqg9OWNzg80H3lDPO+auv85y3T/NHE3GirDk7x/D3sLO57vayw==} + engines: {node: '>= 10'} + cpu: [arm] + os: [android] + + '@napi-rs/wasm-tools-android-arm64@1.0.1': + resolution: {integrity: sha512-WDR7S+aRLV6LtBJAg5fmjKkTZIdrEnnQxgdsb7Cf8pYiMWBHLU+LC49OUVppQ2YSPY0+GeYm9yuZWW3kLjJ7Bg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@napi-rs/wasm-tools-darwin-arm64@1.0.1': + resolution: {integrity: sha512-qWTI+EEkiN0oIn/N2gQo7+TVYil+AJ20jjuzD2vATS6uIjVz+Updeqmszi7zq7rdFTLp6Ea3/z4kDKIfZwmR9g==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@napi-rs/wasm-tools-darwin-x64@1.0.1': + resolution: {integrity: sha512-bA6hubqtHROR5UI3tToAF/c6TDmaAgF0SWgo4rADHtQ4wdn0JeogvOk50gs2TYVhKPE2ZD2+qqt7oBKB+sxW3A==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@napi-rs/wasm-tools-freebsd-x64@1.0.1': + resolution: {integrity: sha512-90+KLBkD9hZEjPQW1MDfwSt5J1L46EUKacpCZWyRuL6iIEO5CgWU0V/JnEgFsDOGyyYtiTvHc5bUdUTWd4I9Vg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [freebsd] + + '@napi-rs/wasm-tools-linux-arm64-gnu@1.0.1': + resolution: {integrity: sha512-rG0QlS65x9K/u3HrKafDf8cFKj5wV2JHGfl8abWgKew0GVPyp6vfsDweOwHbWAjcHtp2LHi6JHoW80/MTHm52Q==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@napi-rs/wasm-tools-linux-arm64-musl@1.0.1': + resolution: {integrity: sha512-jAasbIvjZXCgX0TCuEFQr+4D6Lla/3AAVx2LmDuMjgG4xoIXzjKWl7c4chuaD+TI+prWT0X6LJcdzFT+ROKGHQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@napi-rs/wasm-tools-linux-x64-gnu@1.0.1': + resolution: {integrity: sha512-Plgk5rPqqK2nocBGajkMVbGm010Z7dnUgq0wtnYRZbzWWxwWcXfZMPa8EYxrK4eE8SzpI7VlZP1tdVsdjgGwMw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@napi-rs/wasm-tools-linux-x64-musl@1.0.1': + resolution: {integrity: sha512-GW7AzGuWxtQkyHknHWYFdR0CHmW6is8rG2Rf4V6GNmMpmwtXt/ItWYWtBe4zqJWycMNazpfZKSw/BpT7/MVCXQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@napi-rs/wasm-tools-wasm32-wasi@1.0.1': + resolution: {integrity: sha512-/nQVSTrqSsn7YdAc2R7Ips/tnw5SPUcl3D7QrXCNGPqjbatIspnaexvaOYNyKMU6xPu+pc0BTnKVmqhlJJCPLA==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + + '@napi-rs/wasm-tools-win32-arm64-msvc@1.0.1': + resolution: {integrity: sha512-PFi7oJIBu5w7Qzh3dwFea3sHRO3pojMsaEnUIy22QvsW+UJfNQwJCryVrpoUt8m4QyZXI+saEq/0r4GwdoHYFQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@napi-rs/wasm-tools-win32-ia32-msvc@1.0.1': + resolution: {integrity: sha512-gXkuYzxQsgkj05Zaq+KQTkHIN83dFAwMcTKa2aQcpYPRImFm2AQzEyLtpXmyCWzJ0F9ZYAOmbSyrNew8/us6bw==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + + '@napi-rs/wasm-tools-win32-x64-msvc@1.0.1': + resolution: {integrity: sha512-rEAf05nol3e3eei2sRButmgXP+6ATgm0/38MKhz9Isne82T4rPIMYsCIFj0kOisaGeVwoi2fnm7O9oWp5YVnYQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@napi-rs/wasm-tools@1.0.1': + resolution: {integrity: sha512-enkZYyuCdo+9jneCPE/0fjIta4wWnvVN9hBo2HuiMpRF0q3lzv1J6b/cl7i0mxZUKhBrV3aCKDBQnCOhwKbPmQ==} + engines: {node: '>= 10'} + + '@octokit/auth-token@6.0.0': + resolution: {integrity: sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w==} + engines: {node: '>= 20'} + + '@octokit/core@7.0.6': + resolution: {integrity: sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==} + engines: {node: '>= 20'} + + '@octokit/endpoint@11.0.3': + resolution: {integrity: sha512-FWFlNxghg4HrXkD3ifYbS/IdL/mDHjh9QcsNyhQjN8dplUoZbejsdpmuqdA76nxj2xoWPs7p8uX2SNr9rYu0Ag==} + engines: {node: '>= 20'} + + '@octokit/graphql@9.0.3': + resolution: {integrity: sha512-grAEuupr/C1rALFnXTv6ZQhFuL1D8G5y8CN04RgrO4FIPMrtm+mcZzFG7dcBm+nq+1ppNixu+Jd78aeJOYxlGA==} + engines: {node: '>= 20'} + + '@octokit/openapi-types@27.0.0': + resolution: {integrity: sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==} + + '@octokit/plugin-paginate-rest@14.0.0': + resolution: {integrity: sha512-fNVRE7ufJiAA3XUrha2omTA39M6IXIc6GIZLvlbsm8QOQCYvpq/LkMNGyFlB1d8hTDzsAXa3OKtybdMAYsV/fw==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/plugin-request-log@6.0.0': + resolution: {integrity: sha512-UkOzeEN3W91/eBq9sPZNQ7sUBvYCqYbrrD8gTbBuGtHEuycE4/awMXcYvx6sVYo7LypPhmQwwpUe4Yyu4QZN5Q==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/plugin-rest-endpoint-methods@17.0.0': + resolution: {integrity: sha512-B5yCyIlOJFPqUUeiD0cnBJwWJO8lkJs5d8+ze9QDP6SvfiXSz1BF+91+0MeI1d2yxgOhU/O+CvtiZ9jSkHhFAw==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/request-error@7.1.0': + resolution: {integrity: sha512-KMQIfq5sOPpkQYajXHwnhjCC0slzCNScLHs9JafXc4RAJI+9f+jNDlBNaIMTvazOPLgb4BnlhGJOTbnN0wIjPw==} + engines: {node: '>= 20'} + + '@octokit/request@10.0.11': + resolution: {integrity: sha512-+s7HUxjfFqOMS9VlIwDffq0MikjSAK0gSpG73W+meAvVAvX4MBrHYTK5Bj3Uot55qFT4gzUtfzE4mGWY4Br8/Q==} + engines: {node: '>= 20'} + + '@octokit/rest@22.0.1': + resolution: {integrity: sha512-Jzbhzl3CEexhnivb1iQ0KJ7s5vvjMWcmRtq5aUsKmKDrRW6z3r84ngmiFKFvpZjpiU/9/S6ITPFRpn5s/3uQJw==} + engines: {node: '>= 20'} + + '@octokit/types@16.0.0': + resolution: {integrity: sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==} + '@oxc-project/types@0.139.0': resolution: {integrity: sha512-r9gHphtCs+1M7J0pw6Sn/hh/Wpa/iQrOOkrNAlVLF/gHq+/CJmHIWKKUUhdWjcD6CIa8idarspCsASiXCXvFUw==} @@ -516,6 +1099,9 @@ packages: '@vitest/utils@4.1.10': resolution: {integrity: sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA==} + argparse@2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + assertion-error@2.0.1: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} @@ -523,12 +1109,34 @@ packages: ast-v8-to-istanbul@1.0.4: resolution: {integrity: sha512-0bC0/4bTSrnwdhU3IsZDwEdojvuPrSg59OYZfKsLRtJZ0u8VBx9DebfqqG8bRdCC0I7vjgxmPi41P0lpkhJHtA==} + before-after-hook@4.0.0: + resolution: {integrity: sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==} + chai@6.2.2: resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} - chownr@3.0.0: - resolution: {integrity: sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==} + chardet@2.2.0: + resolution: {integrity: sha512-rddelWYNPRrXq6PtNEN2S3f6t9ILzvqaN5pVgi4kqt9jHQaXIial9PznB5iSPVlQSLNaaH22ItWz3EJtQ10+OA==} + + chownr@3.0.0: + resolution: {integrity: sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==} + engines: {node: '>=18'} + + cli-width@4.1.0: + resolution: {integrity: sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==} + engines: {node: '>= 12'} + + clipanion@4.0.0-rc.4: + resolution: {integrity: sha512-CXkMQxU6s9GklO/1f714dkKBMu1lopS1WFF0B8o4AxPykR1hpozxSiUZ5ZUeBjfPgCWqbcNOtZVFhB8Lkfp1+Q==} + peerDependencies: + typanion: '*' + + colorette@2.0.20: + resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} + + content-type@2.0.0: + resolution: {integrity: sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==} engines: {node: '>=18'} convert-source-map@2.0.0: @@ -537,13 +1145,33 @@ packages: core-util-is@1.0.3: resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + detect-libc@2.1.2: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} + emnapi@1.11.2: + resolution: {integrity: sha512-iMt/XQc69fFn2EvcU6tm14HmXKwyy0lnABugsQlqp6xFuZIUuO+ONVSg2mz+MTVF8WbC+bic65AvRXdoldALKg==} + peerDependencies: + node-addon-api: '>= 6.1.0' + peerDependenciesMeta: + node-addon-api: + optional: true + es-module-lexer@2.2.0: resolution: {integrity: sha512-3lGxdTXCLfe1MYfTz1y2ksAAUM4NAOP6rPEjxGJVKO7TZ5+tvHCaQWGpC4Y3IXvW3ece0Cz1cIP4FWBxOnGCTQ==} + es-toolkit@1.49.0: + resolution: {integrity: sha512-G5iZ6Pc/FNRY/soKZHC+TxGDD83rHUDXxzaWhGCX44vAv/tMs56WMusnm/KMNK+luUPsgA9U28cGr4RDlSzL2g==} + esbuild@0.28.1: resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} engines: {node: '>=18'} @@ -556,6 +1184,15 @@ packages: resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} engines: {node: '>=12.0.0'} + fast-string-truncated-width@3.0.3: + resolution: {integrity: sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==} + + fast-string-width@3.0.2: + resolution: {integrity: sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg==} + + fast-wrap-ansi@0.2.2: + resolution: {integrity: sha512-7F2Fl+TjRSenLqlU3UjSH0iyqopqoZIu7eZVpEirP2g1GtWa2G/ecEmBdgz31+Mxr+ELclgg6sokpSFIQiZ02Q==} + fdir@6.5.0: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} engines: {node: '>=12.0.0'} @@ -577,6 +1214,10 @@ packages: html-escaper@2.0.2: resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} + iconv-lite@0.7.3: + resolution: {integrity: sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==} + engines: {node: '>=0.10.0'} + immediate@3.0.6: resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==} @@ -601,6 +1242,13 @@ packages: js-tokens@10.0.0: resolution: {integrity: sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==} + js-yaml@4.3.0: + resolution: {integrity: sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==} + hasBin: true + + json-with-bigint@3.5.10: + resolution: {integrity: sha512-Vcx+JVNEBts/xfcoCS69sKrOhOk/3TVlvlT+XzUOefVKnnrbYSCKpDCm10pohsJFtsJVYnwa/cXRZ4eElzaM6w==} + jszip@3.10.1: resolution: {integrity: sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==} @@ -699,6 +1347,13 @@ packages: resolution: {integrity: sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==} engines: {node: '>= 18'} + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + mute-stream@3.0.0: + resolution: {integrity: sha512-dkEJPVvun4FryqBmZ5KhDo0K9iDXAwn08tMLDinNdRBNPcYEDiWYysLcc6k3mjTMlbP9KyylvRpd4wFtwrT9rw==} + engines: {node: ^20.17.0 || >=22.9.0} + nanoid@3.3.16: resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} @@ -739,6 +1394,9 @@ packages: safe-buffer@5.1.2: resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} + safer-buffer@2.1.2: + resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + semver@7.8.5: resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} engines: {node: '>=10'} @@ -750,6 +1408,10 @@ packages: siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -789,6 +1451,9 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + typanion@3.14.0: + resolution: {integrity: sha512-ZW/lVMRabETuYCd9O9ZvMhAh8GslSqaUjxmK/JLPCh6l73CvLBiuXswj/+7LdnWOgYsQ130FqLzFz5aGT4I3Ug==} + typescript@7.0.2: resolution: {integrity: sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==} engines: {node: '>=16.20.0'} @@ -797,6 +1462,9 @@ packages: undici-types@8.3.0: resolution: {integrity: sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==} + universal-user-agent@7.0.3: + resolution: {integrity: sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==} + util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} @@ -916,11 +1584,22 @@ snapshots: tslib: 2.8.1 optional: true + '@emnapi/core@1.11.2': + dependencies: + '@emnapi/wasi-threads': 1.2.2 + tslib: 2.8.1 + optional: true + '@emnapi/runtime@1.11.1': dependencies: tslib: 2.8.1 optional: true + '@emnapi/runtime@1.11.2': + dependencies: + tslib: 2.8.1 + optional: true + '@emnapi/wasi-threads@1.2.2': dependencies: tslib: 2.8.1 @@ -1004,6 +1683,125 @@ snapshots: '@esbuild/win32-x64@0.28.1': optional: true + '@inquirer/ansi@2.0.7': {} + + '@inquirer/checkbox@5.2.1(@types/node@26.1.1)': + dependencies: + '@inquirer/ansi': 2.0.7 + '@inquirer/core': 11.2.1(@types/node@26.1.1) + '@inquirer/figures': 2.0.7 + '@inquirer/type': 4.0.7(@types/node@26.1.1) + optionalDependencies: + '@types/node': 26.1.1 + + '@inquirer/confirm@6.1.1(@types/node@26.1.1)': + dependencies: + '@inquirer/core': 11.2.1(@types/node@26.1.1) + '@inquirer/type': 4.0.7(@types/node@26.1.1) + optionalDependencies: + '@types/node': 26.1.1 + + '@inquirer/core@11.2.1(@types/node@26.1.1)': + dependencies: + '@inquirer/ansi': 2.0.7 + '@inquirer/figures': 2.0.7 + '@inquirer/type': 4.0.7(@types/node@26.1.1) + cli-width: 4.1.0 + fast-wrap-ansi: 0.2.2 + mute-stream: 3.0.0 + signal-exit: 4.1.0 + optionalDependencies: + '@types/node': 26.1.1 + + '@inquirer/editor@5.2.2(@types/node@26.1.1)': + dependencies: + '@inquirer/core': 11.2.1(@types/node@26.1.1) + '@inquirer/external-editor': 3.0.3(@types/node@26.1.1) + '@inquirer/type': 4.0.7(@types/node@26.1.1) + optionalDependencies: + '@types/node': 26.1.1 + + '@inquirer/expand@5.1.1(@types/node@26.1.1)': + dependencies: + '@inquirer/core': 11.2.1(@types/node@26.1.1) + '@inquirer/type': 4.0.7(@types/node@26.1.1) + optionalDependencies: + '@types/node': 26.1.1 + + '@inquirer/external-editor@3.0.3(@types/node@26.1.1)': + dependencies: + chardet: 2.2.0 + iconv-lite: 0.7.3 + optionalDependencies: + '@types/node': 26.1.1 + + '@inquirer/figures@2.0.7': {} + + '@inquirer/input@5.1.2(@types/node@26.1.1)': + dependencies: + '@inquirer/core': 11.2.1(@types/node@26.1.1) + '@inquirer/type': 4.0.7(@types/node@26.1.1) + optionalDependencies: + '@types/node': 26.1.1 + + '@inquirer/number@4.1.1(@types/node@26.1.1)': + dependencies: + '@inquirer/core': 11.2.1(@types/node@26.1.1) + '@inquirer/type': 4.0.7(@types/node@26.1.1) + optionalDependencies: + '@types/node': 26.1.1 + + '@inquirer/password@5.1.1(@types/node@26.1.1)': + dependencies: + '@inquirer/ansi': 2.0.7 + '@inquirer/core': 11.2.1(@types/node@26.1.1) + '@inquirer/type': 4.0.7(@types/node@26.1.1) + optionalDependencies: + '@types/node': 26.1.1 + + '@inquirer/prompts@8.5.2(@types/node@26.1.1)': + dependencies: + '@inquirer/checkbox': 5.2.1(@types/node@26.1.1) + '@inquirer/confirm': 6.1.1(@types/node@26.1.1) + '@inquirer/editor': 5.2.2(@types/node@26.1.1) + '@inquirer/expand': 5.1.1(@types/node@26.1.1) + '@inquirer/input': 5.1.2(@types/node@26.1.1) + '@inquirer/number': 4.1.1(@types/node@26.1.1) + '@inquirer/password': 5.1.1(@types/node@26.1.1) + '@inquirer/rawlist': 5.3.1(@types/node@26.1.1) + '@inquirer/search': 4.2.1(@types/node@26.1.1) + '@inquirer/select': 5.2.1(@types/node@26.1.1) + optionalDependencies: + '@types/node': 26.1.1 + + '@inquirer/rawlist@5.3.1(@types/node@26.1.1)': + dependencies: + '@inquirer/core': 11.2.1(@types/node@26.1.1) + '@inquirer/type': 4.0.7(@types/node@26.1.1) + optionalDependencies: + '@types/node': 26.1.1 + + '@inquirer/search@4.2.1(@types/node@26.1.1)': + dependencies: + '@inquirer/core': 11.2.1(@types/node@26.1.1) + '@inquirer/figures': 2.0.7 + '@inquirer/type': 4.0.7(@types/node@26.1.1) + optionalDependencies: + '@types/node': 26.1.1 + + '@inquirer/select@5.2.1(@types/node@26.1.1)': + dependencies: + '@inquirer/ansi': 2.0.7 + '@inquirer/core': 11.2.1(@types/node@26.1.1) + '@inquirer/figures': 2.0.7 + '@inquirer/type': 4.0.7(@types/node@26.1.1) + optionalDependencies: + '@types/node': 26.1.1 + + '@inquirer/type@4.0.7(@types/node@26.1.1)': + optionalDependencies: + '@types/node': 26.1.1 + '@isaacs/fs-minipass@4.0.1': dependencies: minipass: 7.1.3 @@ -1018,6 +1816,192 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@napi-rs/cli@3.7.4(@emnapi/core@1.11.2)(@emnapi/runtime@1.11.2)(@types/node@26.1.1)': + dependencies: + '@inquirer/prompts': 8.5.2(@types/node@26.1.1) + '@napi-rs/cross-toolchain': 1.0.3 + '@napi-rs/wasm-tools': 1.0.1(@emnapi/core@1.11.2)(@emnapi/runtime@1.11.2) + '@octokit/rest': 22.0.1 + clipanion: 4.0.0-rc.4(typanion@3.14.0) + colorette: 2.0.20 + emnapi: 1.11.2 + es-toolkit: 1.49.0 + js-yaml: 4.3.0 + obug: 2.1.3 + semver: 7.8.5 + typanion: 3.14.0 + optionalDependencies: + '@emnapi/runtime': 1.11.2 + transitivePeerDependencies: + - '@emnapi/core' + - '@napi-rs/cross-toolchain-arm64-target-aarch64' + - '@napi-rs/cross-toolchain-arm64-target-armv7' + - '@napi-rs/cross-toolchain-arm64-target-ppc64le' + - '@napi-rs/cross-toolchain-arm64-target-s390x' + - '@napi-rs/cross-toolchain-arm64-target-x86_64' + - '@napi-rs/cross-toolchain-x64-target-aarch64' + - '@napi-rs/cross-toolchain-x64-target-armv7' + - '@napi-rs/cross-toolchain-x64-target-ppc64le' + - '@napi-rs/cross-toolchain-x64-target-s390x' + - '@napi-rs/cross-toolchain-x64-target-x86_64' + - '@types/node' + - node-addon-api + - supports-color + + '@napi-rs/cross-toolchain@1.0.3': + dependencies: + '@napi-rs/lzma': 1.5.1 + '@napi-rs/tar': 1.1.1 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + '@napi-rs/lzma-android-arm-eabi@1.5.1': + optional: true + + '@napi-rs/lzma-android-arm64@1.5.1': + optional: true + + '@napi-rs/lzma-darwin-arm64@1.5.1': + optional: true + + '@napi-rs/lzma-darwin-x64@1.5.1': + optional: true + + '@napi-rs/lzma-freebsd-x64@1.5.1': + optional: true + + '@napi-rs/lzma-linux-arm-gnueabihf@1.5.1': + optional: true + + '@napi-rs/lzma-linux-arm64-gnu@1.5.1': + optional: true + + '@napi-rs/lzma-linux-arm64-musl@1.5.1': + optional: true + + '@napi-rs/lzma-linux-ppc64-gnu@1.5.1': + optional: true + + '@napi-rs/lzma-linux-riscv64-gnu@1.5.1': + optional: true + + '@napi-rs/lzma-linux-s390x-gnu@1.5.1': + optional: true + + '@napi-rs/lzma-linux-x64-gnu@1.5.1': + optional: true + + '@napi-rs/lzma-linux-x64-musl@1.5.1': + optional: true + + '@napi-rs/lzma-wasm32-wasi@1.5.1': + dependencies: + '@emnapi/core': 1.11.2 + '@emnapi/runtime': 1.11.2 + '@napi-rs/wasm-runtime': 1.1.6(@emnapi/core@1.11.2)(@emnapi/runtime@1.11.2) + optional: true + + '@napi-rs/lzma-win32-arm64-msvc@1.5.1': + optional: true + + '@napi-rs/lzma-win32-ia32-msvc@1.5.1': + optional: true + + '@napi-rs/lzma-win32-x64-msvc@1.5.1': + optional: true + + '@napi-rs/lzma@1.5.1': + optionalDependencies: + '@napi-rs/lzma-android-arm-eabi': 1.5.1 + '@napi-rs/lzma-android-arm64': 1.5.1 + '@napi-rs/lzma-darwin-arm64': 1.5.1 + '@napi-rs/lzma-darwin-x64': 1.5.1 + '@napi-rs/lzma-freebsd-x64': 1.5.1 + '@napi-rs/lzma-linux-arm-gnueabihf': 1.5.1 + '@napi-rs/lzma-linux-arm64-gnu': 1.5.1 + '@napi-rs/lzma-linux-arm64-musl': 1.5.1 + '@napi-rs/lzma-linux-ppc64-gnu': 1.5.1 + '@napi-rs/lzma-linux-riscv64-gnu': 1.5.1 + '@napi-rs/lzma-linux-s390x-gnu': 1.5.1 + '@napi-rs/lzma-linux-x64-gnu': 1.5.1 + '@napi-rs/lzma-linux-x64-musl': 1.5.1 + '@napi-rs/lzma-wasm32-wasi': 1.5.1 + '@napi-rs/lzma-win32-arm64-msvc': 1.5.1 + '@napi-rs/lzma-win32-ia32-msvc': 1.5.1 + '@napi-rs/lzma-win32-x64-msvc': 1.5.1 + + '@napi-rs/tar-android-arm-eabi@1.1.1': + optional: true + + '@napi-rs/tar-android-arm64@1.1.1': + optional: true + + '@napi-rs/tar-darwin-arm64@1.1.1': + optional: true + + '@napi-rs/tar-darwin-x64@1.1.1': + optional: true + + '@napi-rs/tar-freebsd-x64@1.1.1': + optional: true + + '@napi-rs/tar-linux-arm-gnueabihf@1.1.1': + optional: true + + '@napi-rs/tar-linux-arm64-gnu@1.1.1': + optional: true + + '@napi-rs/tar-linux-arm64-musl@1.1.1': + optional: true + + '@napi-rs/tar-linux-ppc64-gnu@1.1.1': + optional: true + + '@napi-rs/tar-linux-s390x-gnu@1.1.1': + optional: true + + '@napi-rs/tar-linux-x64-gnu@1.1.1': + optional: true + + '@napi-rs/tar-linux-x64-musl@1.1.1': + optional: true + + '@napi-rs/tar-wasm32-wasi@1.1.1': + dependencies: + '@emnapi/core': 1.11.2 + '@emnapi/runtime': 1.11.2 + '@napi-rs/wasm-runtime': 1.1.6(@emnapi/core@1.11.2)(@emnapi/runtime@1.11.2) + optional: true + + '@napi-rs/tar-win32-arm64-msvc@1.1.1': + optional: true + + '@napi-rs/tar-win32-ia32-msvc@1.1.1': + optional: true + + '@napi-rs/tar-win32-x64-msvc@1.1.1': + optional: true + + '@napi-rs/tar@1.1.1': + optionalDependencies: + '@napi-rs/tar-android-arm-eabi': 1.1.1 + '@napi-rs/tar-android-arm64': 1.1.1 + '@napi-rs/tar-darwin-arm64': 1.1.1 + '@napi-rs/tar-darwin-x64': 1.1.1 + '@napi-rs/tar-freebsd-x64': 1.1.1 + '@napi-rs/tar-linux-arm-gnueabihf': 1.1.1 + '@napi-rs/tar-linux-arm64-gnu': 1.1.1 + '@napi-rs/tar-linux-arm64-musl': 1.1.1 + '@napi-rs/tar-linux-ppc64-gnu': 1.1.1 + '@napi-rs/tar-linux-s390x-gnu': 1.1.1 + '@napi-rs/tar-linux-x64-gnu': 1.1.1 + '@napi-rs/tar-linux-x64-musl': 1.1.1 + '@napi-rs/tar-wasm32-wasi': 1.1.1 + '@napi-rs/tar-win32-arm64-msvc': 1.1.1 + '@napi-rs/tar-win32-ia32-msvc': 1.1.1 + '@napi-rs/tar-win32-x64-msvc': 1.1.1 + '@napi-rs/wasm-runtime@1.1.6(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1)': dependencies: '@emnapi/core': 1.11.1 @@ -1025,6 +2009,139 @@ snapshots: '@tybys/wasm-util': 0.10.3 optional: true + '@napi-rs/wasm-runtime@1.1.6(@emnapi/core@1.11.2)(@emnapi/runtime@1.11.2)': + dependencies: + '@emnapi/core': 1.11.2 + '@emnapi/runtime': 1.11.2 + '@tybys/wasm-util': 0.10.3 + optional: true + + '@napi-rs/wasm-tools-android-arm-eabi@1.0.1': + optional: true + + '@napi-rs/wasm-tools-android-arm64@1.0.1': + optional: true + + '@napi-rs/wasm-tools-darwin-arm64@1.0.1': + optional: true + + '@napi-rs/wasm-tools-darwin-x64@1.0.1': + optional: true + + '@napi-rs/wasm-tools-freebsd-x64@1.0.1': + optional: true + + '@napi-rs/wasm-tools-linux-arm64-gnu@1.0.1': + optional: true + + '@napi-rs/wasm-tools-linux-arm64-musl@1.0.1': + optional: true + + '@napi-rs/wasm-tools-linux-x64-gnu@1.0.1': + optional: true + + '@napi-rs/wasm-tools-linux-x64-musl@1.0.1': + optional: true + + '@napi-rs/wasm-tools-wasm32-wasi@1.0.1(@emnapi/core@1.11.2)(@emnapi/runtime@1.11.2)': + dependencies: + '@napi-rs/wasm-runtime': 1.1.6(@emnapi/core@1.11.2)(@emnapi/runtime@1.11.2) + transitivePeerDependencies: + - '@emnapi/core' + - '@emnapi/runtime' + optional: true + + '@napi-rs/wasm-tools-win32-arm64-msvc@1.0.1': + optional: true + + '@napi-rs/wasm-tools-win32-ia32-msvc@1.0.1': + optional: true + + '@napi-rs/wasm-tools-win32-x64-msvc@1.0.1': + optional: true + + '@napi-rs/wasm-tools@1.0.1(@emnapi/core@1.11.2)(@emnapi/runtime@1.11.2)': + optionalDependencies: + '@napi-rs/wasm-tools-android-arm-eabi': 1.0.1 + '@napi-rs/wasm-tools-android-arm64': 1.0.1 + '@napi-rs/wasm-tools-darwin-arm64': 1.0.1 + '@napi-rs/wasm-tools-darwin-x64': 1.0.1 + '@napi-rs/wasm-tools-freebsd-x64': 1.0.1 + '@napi-rs/wasm-tools-linux-arm64-gnu': 1.0.1 + '@napi-rs/wasm-tools-linux-arm64-musl': 1.0.1 + '@napi-rs/wasm-tools-linux-x64-gnu': 1.0.1 + '@napi-rs/wasm-tools-linux-x64-musl': 1.0.1 + '@napi-rs/wasm-tools-wasm32-wasi': 1.0.1(@emnapi/core@1.11.2)(@emnapi/runtime@1.11.2) + '@napi-rs/wasm-tools-win32-arm64-msvc': 1.0.1 + '@napi-rs/wasm-tools-win32-ia32-msvc': 1.0.1 + '@napi-rs/wasm-tools-win32-x64-msvc': 1.0.1 + transitivePeerDependencies: + - '@emnapi/core' + - '@emnapi/runtime' + + '@octokit/auth-token@6.0.0': {} + + '@octokit/core@7.0.6': + dependencies: + '@octokit/auth-token': 6.0.0 + '@octokit/graphql': 9.0.3 + '@octokit/request': 10.0.11 + '@octokit/request-error': 7.1.0 + '@octokit/types': 16.0.0 + before-after-hook: 4.0.0 + universal-user-agent: 7.0.3 + + '@octokit/endpoint@11.0.3': + dependencies: + '@octokit/types': 16.0.0 + universal-user-agent: 7.0.3 + + '@octokit/graphql@9.0.3': + dependencies: + '@octokit/request': 10.0.11 + '@octokit/types': 16.0.0 + universal-user-agent: 7.0.3 + + '@octokit/openapi-types@27.0.0': {} + + '@octokit/plugin-paginate-rest@14.0.0(@octokit/core@7.0.6)': + dependencies: + '@octokit/core': 7.0.6 + '@octokit/types': 16.0.0 + + '@octokit/plugin-request-log@6.0.0(@octokit/core@7.0.6)': + dependencies: + '@octokit/core': 7.0.6 + + '@octokit/plugin-rest-endpoint-methods@17.0.0(@octokit/core@7.0.6)': + dependencies: + '@octokit/core': 7.0.6 + '@octokit/types': 16.0.0 + + '@octokit/request-error@7.1.0': + dependencies: + '@octokit/types': 16.0.0 + + '@octokit/request@10.0.11': + dependencies: + '@octokit/endpoint': 11.0.3 + '@octokit/request-error': 7.1.0 + '@octokit/types': 16.0.0 + content-type: 2.0.0 + json-with-bigint: 3.5.10 + universal-user-agent: 7.0.3 + + '@octokit/rest@22.0.1': + dependencies: + '@octokit/core': 7.0.6 + '@octokit/plugin-paginate-rest': 14.0.0(@octokit/core@7.0.6) + '@octokit/plugin-request-log': 6.0.0(@octokit/core@7.0.6) + '@octokit/plugin-rest-endpoint-methods': 17.0.0(@octokit/core@7.0.6) + + '@octokit/types@16.0.0': + dependencies: + '@octokit/openapi-types': 27.0.0 + '@oxc-project/types@0.139.0': {} '@rolldown/binding-android-arm64@1.1.5': @@ -1213,6 +2330,8 @@ snapshots: convert-source-map: 2.0.0 tinyrainbow: 3.1.0 + argparse@2.0.1: {} + assertion-error@2.0.1: {} ast-v8-to-istanbul@1.0.4: @@ -1221,20 +2340,42 @@ snapshots: estree-walker: 3.0.3 js-tokens: 10.0.0 + before-after-hook@4.0.0: {} + chai@6.2.2: {} + chardet@2.2.0: {} + chownr@3.0.0: optional: true + cli-width@4.1.0: {} + + clipanion@4.0.0-rc.4(typanion@3.14.0): + dependencies: + typanion: 3.14.0 + + colorette@2.0.20: {} + + content-type@2.0.0: {} + convert-source-map@2.0.0: {} core-util-is@1.0.3: optional: true + debug@4.4.3: + dependencies: + ms: 2.1.3 + detect-libc@2.1.2: {} + emnapi@1.11.2: {} + es-module-lexer@2.2.0: {} + es-toolkit@1.49.0: {} + esbuild@0.28.1: optionalDependencies: '@esbuild/aix-ppc64': 0.28.1 @@ -1271,6 +2412,16 @@ snapshots: expect-type@1.4.0: {} + fast-string-truncated-width@3.0.3: {} + + fast-string-width@3.0.2: + dependencies: + fast-string-truncated-width: 3.0.3 + + fast-wrap-ansi@0.2.2: + dependencies: + fast-string-width: 3.0.2 + fdir@6.5.0(picomatch@4.0.5): optionalDependencies: picomatch: 4.0.5 @@ -1282,6 +2433,10 @@ snapshots: html-escaper@2.0.2: {} + iconv-lite@0.7.3: + dependencies: + safer-buffer: 2.1.2 + immediate@3.0.6: optional: true @@ -1306,6 +2461,12 @@ snapshots: js-tokens@10.0.0: {} + js-yaml@4.3.0: + dependencies: + argparse: 2.0.1 + + json-with-bigint@3.5.10: {} + jszip@3.10.1: dependencies: lie: 3.3.0 @@ -1390,6 +2551,10 @@ snapshots: minipass: 7.1.3 optional: true + ms@2.1.3: {} + + mute-stream@3.0.0: {} + nanoid@3.3.16: {} obug@2.1.3: {} @@ -1447,6 +2612,8 @@ snapshots: safe-buffer@5.1.2: optional: true + safer-buffer@2.1.2: {} + semver@7.8.5: {} setimmediate@1.0.5: @@ -1454,6 +2621,8 @@ snapshots: siginfo@2.0.0: {} + signal-exit@4.1.0: {} + source-map-js@1.2.1: {} stackback@0.0.2: {} @@ -1492,6 +2661,8 @@ snapshots: tslib@2.8.1: optional: true + typanion@3.14.0: {} + typescript@7.0.2: optionalDependencies: '@typescript/typescript-aix-ppc64': 7.0.2 @@ -1517,6 +2688,8 @@ snapshots: undici-types@8.3.0: {} + universal-user-agent@7.0.3: {} + util-deprecate@1.0.2: optional: true diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 4bebc12..1de7f9c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,4 +1,8 @@ -packages: [] +packages: + - native + - npm/* + +linkWorkspacePackages: true allowBuilds: esbuild: true diff --git a/scripts/append-release-proof.mjs b/scripts/append-release-proof.mjs new file mode 100644 index 0000000..50f0e29 --- /dev/null +++ b/scripts/append-release-proof.mjs @@ -0,0 +1,33 @@ +import { execFileSync } from "node:child_process"; +import { appendFileSync, readFileSync } from "node:fs"; + +const [notesPath, manifestPath, repository, runId] = process.argv.slice(2); +if (!notesPath || !manifestPath || !repository || !runId) { + throw new Error("usage: append-release-proof "); +} +const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); +const lines = [ + "", + "### Published packages", + "", + "| Package | Registry tarball | Integrity | Provenance |", + "|---|---|---|---|", +]; +for (const artifact of manifest) { + const spec = `${artifact.name}@${artifact.version}`; + const dist = JSON.parse(execFileSync("npm", ["view", spec, "dist", "--json"], { + encoding: "utf8", + })); + if (dist.integrity !== artifact.integrity || !dist.tarball || !dist.attestations?.url) { + throw new Error(`${spec} registry proof is incomplete or does not match the release artifact`); + } + lines.push( + `| [${spec}](https://www.npmjs.com/package/${artifact.name}/v/${artifact.version}) | [tgz](${dist.tarball}) | \`${dist.integrity}\` | [attestation](${dist.attestations.url}) |`, + ); +} +lines.push( + "", + `Release proof: [GitHub Actions run](https://github.com/${repository}/actions/runs/${runId}).`, + "", +); +appendFileSync(notesPath, `${lines.join("\n")}\n`); diff --git a/scripts/bench-publish.mjs b/scripts/bench-publish.mjs new file mode 100644 index 0000000..7a34150 --- /dev/null +++ b/scripts/bench-publish.mjs @@ -0,0 +1,165 @@ +import { createRequire } from "node:module"; +import fsSync from "node:fs"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import { publishFileExclusive } from "../dist/publish-file.js"; +import { + __resetFsSafeNativeConfigForTest, + configureFsSafeNative, +} from "../dist/native-config.js"; +import { + __resetNativeLoaderForTest, + __setNativeLoaderForTest, +} from "../dist/native.js"; + +const require = createRequire(import.meta.url); +const native = require("../native"); +const sizes = [ + { label: "4 KiB", bytes: 4 * 1024, iterations: 7 }, + { label: "1 MiB", bytes: 1024 * 1024, iterations: 7 }, + { label: "64 MiB", bytes: 64 * 1024 * 1024, iterations: 5 }, +]; +const root = await fs.mkdtemp(path.join(os.tmpdir(), "fs-safe-bench-publish-")); +const filesystem = await fs.statfs(root); +const filesystemName = + process.platform === "darwin" && filesystem.type === 26 + ? "APFS" + : `type-${Number(filesystem.type).toString(16)}`; +const originalLink = fs.link; +const rows = []; + +try { + for (const fixture of sizes) { + const sourcePath = path.join(root, `source-${fixture.bytes}`); + const source = await fs.open(sourcePath, "w", 0o600); + try { + const chunk = Buffer.alloc(Math.min(fixture.bytes, 1024 * 1024), 0x5a); + for (let position = 0; position < fixture.bytes; ) { + const length = Math.min(chunk.length, fixture.bytes - position); + const { bytesWritten } = await source.write(chunk, 0, length, position); + if (bytesWritten <= 0) throw new Error("benchmark fixture write made no progress"); + position += bytesWritten; + } + await source.sync(); + } finally { + await source.close(); + } + + for (const mode of ["javascript", "native"]) { + let selectedTier = mode === "javascript" ? "js-loop" : undefined; + __resetFsSafeNativeConfigForTest(); + __resetNativeLoaderForTest(); + if (mode === "javascript") { + configureFsSafeNative({ mode: "off" }); + fs.link = async () => { + throw Object.assign(new Error("benchmark forces copy fallback"), { code: "EXDEV" }); + }; + } else { + fs.link = originalLink; + __setNativeLoaderForTest(() => ({ + ...native, + linkBeneath() { + throw Object.assign(new Error("benchmark forces copy fallback"), { code: "EXDEV" }); + }, + cloneFileExclusive(...args) { + const fd = native.cloneFileExclusive(...args); + selectedTier = "clone"; + return fd; + }, + async copyFileRangeExclusive(...args) { + const result = await native.copyFileRangeExclusive(...args); + if (!result.errorCode) selectedTier = "copy_file_range"; + return result; + }, + })); + configureFsSafeNative({ mode: "require" }); + } + + const samples = []; + const tiers = new Set(); + for (let iteration = 0; iteration < fixture.iterations; iteration += 1) { + selectedTier = mode === "javascript" ? "js-loop" : undefined; + const targetPath = path.join(root, `target-${mode}-${fixture.bytes}-${iteration}`); + const started = performance.now(); + const result = await publishFileExclusive({ + sourcePath, + targetPath, + strategy: "link-or-copy", + }); + samples.push(performance.now() - started); + if (result.method !== "exclusive-copy") { + throw new Error(`unexpected publication method: ${result.method}`); + } + tiers.add(selectedTier ?? "js-loop"); + const stat = await fs.stat(targetPath); + if (stat.size !== fixture.bytes) throw new Error("benchmark publication size mismatch"); + await fs.rm(targetPath); + } + samples.sort((a, b) => a - b); + const medianMs = samples[Math.floor(samples.length / 2)]; + rows.push({ + size: fixture.label, + mode, + tier: [...tiers].join(" -> "), + medianMs: Number(medianMs.toFixed(2)), + throughputMiBs: Number( + ((fixture.bytes / (1024 * 1024)) / (medianMs / 1000)).toFixed(1), + ), + }); + } + + if (process.platform === "darwin") { + const source = await fs.open(sourcePath, "r"); + const directory = await fs.open(root, fsSync.constants.O_RDONLY | fsSync.constants.O_DIRECTORY); + const samples = []; + try { + for (let iteration = 0; iteration < fixture.iterations; iteration += 1) { + const targetName = `target-clone-primitive-${fixture.bytes}-${iteration}`; + const started = performance.now(); + const fd = native.cloneFileExclusive(source.fd, directory.fd, targetName); + samples.push(performance.now() - started); + fsSync.closeSync(fd); + await fs.rm(path.join(root, targetName)); + } + } finally { + await directory.close(); + await source.close(); + } + samples.sort((a, b) => a - b); + const medianMs = samples[Math.floor(samples.length / 2)]; + rows.push({ + size: fixture.label, + mode: "native primitive", + tier: "clone", + medianMs: Number(medianMs.toFixed(2)), + throughputMiBs: null, + }); + } + } +} finally { + fs.link = originalLink; + __resetFsSafeNativeConfigForTest(); + __resetNativeLoaderForTest(); + await fs.rm(root, { recursive: true, force: true }); +} + +console.log( + `publish benchmark (${process.platform}-${process.arch}, Node ${process.version}, ${filesystemName}, ${os.tmpdir()})`, +); +console.log("| Size | Path | Exercised tier | Median ms | MiB/s |"); +console.log("|---:|:---|:---|---:|---:|"); +for (const row of rows) { + const throughput = row.throughputMiBs === null ? "n/a" : row.throughputMiBs.toFixed(1); + console.log(`| ${row.size} | ${row.mode} | ${row.tier} | ${row.medianMs.toFixed(2)} | ${throughput} |`); +} +console.log( + JSON.stringify({ + platform: `${process.platform}-${process.arch}`, + node: process.version, + filesystem: filesystemName, + tempRoot: os.tmpdir(), + rows, + }), +); diff --git a/scripts/build-docs-site.mjs b/scripts/build-docs-site.mjs index 019209b..99d27af 100644 --- a/scripts/build-docs-site.mjs +++ b/scripts/build-docs-site.mjs @@ -26,7 +26,7 @@ const productDescription = const installCmd = "pnpm add @openclaw/fs-safe"; const sections = [ - ["Start", ["index.md", "install.md", "quickstart.md", "security-model.md", "python-helper.md", "config.md"]], + ["Start", ["index.md", "install.md", "quickstart.md", "security-model.md", "native-helper.md", "native.md", "config.md"]], ["Root API", ["root.md", "reading.md", "writing.md", "path-scope.md"]], ["Atomic & temp", ["atomic.md", "output.md", "json.md", "temp.md", "archive.md"]], ["Stores", ["store.md", "json-store.md", "file-store.md", "private-file-store.md"]], diff --git a/scripts/check-file-size.mjs b/scripts/check-file-size.mjs index 9354a62..9a35075 100644 --- a/scripts/check-file-size.mjs +++ b/scripts/check-file-size.mjs @@ -5,7 +5,6 @@ const DEFAULT_MAX_LINES = 500; const LINE_BUDGETS = new Map([ ["src/file-store.ts", 585], ["src/permissions.ts", 566], - ["src/pinned-python.ts", 750], ["src/root-impl.ts", 1750], ["src/root-path.ts", 862], ["test/api-coverage.test.ts", 983], diff --git a/scripts/check-release-packages.mjs b/scripts/check-release-packages.mjs new file mode 100644 index 0000000..36ca127 --- /dev/null +++ b/scripts/check-release-packages.mjs @@ -0,0 +1,136 @@ +import { execFileSync } from "node:child_process"; +import { + existsSync, + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +const outputIndex = process.argv.indexOf("--output"); +const outputDir = resolve(outputIndex >= 0 ? process.argv[outputIndex + 1] : "release-artifacts"); +mkdirSync(outputDir, { recursive: true }); + +const packageDirs = [ + "npm/linux-x64-gnu", + "npm/linux-x64-musl", + "npm/linux-arm64-gnu", + "npm/linux-arm64-musl", + "npm/darwin-x64", + "npm/darwin-arm64", + "npm/win32-x64-msvc", + "native", + ".", +]; + +const packages = packageDirs.map((directory) => ({ + directory, + packageJson: JSON.parse(readFileSync(join(directory, "package.json"), "utf8")), +})); +const rootVersion = packages.at(-1).packageJson.version; + +function normalizeRepository(repository) { + const value = typeof repository === "string" ? repository : repository?.url; + return String(value ?? "") + .replace(/^github:/u, "https://github.com/") + .replace(/^git\+/u, "") + .replace(/\.git$/iu, "") + .replace(/\/+$/u, ""); +} + +for (const entry of packages) { + const pkg = entry.packageJson; + if (pkg.version !== rootVersion) { + throw new Error(`${pkg.name} version ${pkg.version} does not match ${rootVersion}`); + } + if (pkg.author !== "OpenClaw Team ") { + throw new Error(`${pkg.name} has unexpected author metadata`); + } + if (normalizeRepository(pkg.repository) !== "https://github.com/openclaw/fs-safe") { + throw new Error(`${pkg.name} has unexpected repository metadata`); + } + if (pkg.publishConfig?.access !== "public" || pkg.publishConfig?.provenance !== true) { + throw new Error(`${pkg.name} must publish publicly with provenance`); + } +} + +const nativePackage = packages.find((entry) => entry.directory === "native").packageJson; +for (const [name, version] of Object.entries(nativePackage.optionalDependencies ?? {})) { + if (version !== rootVersion || !packages.some((entry) => entry.packageJson.name === name)) { + throw new Error(`native optional dependency ${name}@${version} is not a release package`); + } +} +const rootPackage = packages.at(-1).packageJson; +if (rootPackage.optionalDependencies?.["@openclaw/fs-safe-native"] !== rootVersion) { + throw new Error("root package must depend on the matching native package version"); +} + +const manifest = []; +for (const entry of packages) { + const packed = JSON.parse(execFileSync( + "npm", + ["pack", "--json", "--ignore-scripts", "--pack-destination", outputDir], + { cwd: resolve(entry.directory), encoding: "utf8" }, + )); + const [artifact] = packed; + if (!artifact?.filename || !artifact?.integrity || artifact.version !== rootVersion) { + throw new Error(`npm pack returned incomplete metadata for ${entry.packageJson.name}`); + } + const paths = new Set(artifact.files.map((file) => file.path)); + if (entry.directory.startsWith("npm/")) { + const binary = entry.packageJson.main; + if (!paths.has(binary) || !existsSync(join(entry.directory, binary))) { + throw new Error(`${entry.packageJson.name} is missing ${binary}`); + } + if (statSync(join(entry.directory, binary)).size === 0) { + throw new Error(`${entry.packageJson.name} contains an empty native binary`); + } + } else if (entry.directory === "native") { + for (const expected of ["index.js", "index.d.ts", "package.json"]) { + if (!paths.has(expected)) throw new Error(`${entry.packageJson.name} is missing ${expected}`); + } + } else { + for (const expected of ["dist/index.js", "dist/index.d.ts", "package.json"]) { + if (!paths.has(expected)) throw new Error(`${entry.packageJson.name} is missing ${expected}`); + } + } + manifest.push({ + name: entry.packageJson.name, + version: rootVersion, + filename: artifact.filename, + integrity: artifact.integrity, + }); +} + +writeFileSync(join(outputDir, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`); + +const smoke = mkdtempSync(join(tmpdir(), "fs-safe-release-smoke-")); +try { + writeFileSync(join(smoke, "package.json"), '{"private":true,"type":"module"}\n'); + const wanted = [ + "@openclaw/fs-safe", + "@openclaw/fs-safe-native", + "@openclaw/fs-safe-native-linux-x64-gnu", + ].map((name) => join(outputDir, manifest.find((entry) => entry.name === name).filename)); + execFileSync( + "npm", + ["install", "--force", "--ignore-scripts", "--no-audit", "--no-fund", "--no-package-lock", ...wanted], + { cwd: smoke, stdio: "pipe" }, + ); + execFileSync( + process.execPath, + ["--input-type=module", "--eval", "await import('@openclaw/fs-safe'); await import('@openclaw/fs-safe/config');"], + { cwd: smoke, stdio: "pipe" }, + ); + execFileSync( + process.execPath, + ["--eval", "const n=require('@openclaw/fs-safe-native'); if(typeof n.openBeneath!=='function') process.exit(1)"], + { cwd: smoke, stdio: "pipe" }, + ); +} finally { + rmSync(smoke, { recursive: true, force: true }); +} diff --git a/scripts/harden-native-loader.mjs b/scripts/harden-native-loader.mjs new file mode 100644 index 0000000..b0ba232 --- /dev/null +++ b/scripts/harden-native-loader.mjs @@ -0,0 +1,41 @@ +import { readFileSync, writeFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +const root = fileURLToPath(new URL("../", import.meta.url)); +const loaderPath = fileURLToPath(new URL("../native/index.js", import.meta.url)); +const typesPath = fileURLToPath(new URL("../native/index.d.ts", import.meta.url)); +const templatePath = fileURLToPath(new URL("./native-loader.cjs", import.meta.url)); +const loader = readFileSync(loaderPath, "utf8"); +const hardened = readFileSync(templatePath, "utf8"); + +if (hardened !== loader) { + writeFileSync(loaderPath, hardened); + console.log(`hardened ${loaderPath.slice(root.length)}`); +} + +const generatedTypes = readFileSync(typesPath, "utf8"); +const hardenedTypes = generatedTypes + .replace( + /inspectArchiveNative\(([^)]*)\): Promise/, + "inspectArchiveNative($1): Promise>", + ) + .replace( + /extractArchiveNative\(([^)]*)\): Promise/, + "extractArchiveNative($1): Promise", + ) + .replace( + /readArchiveEntryNative\(([^)]*)\): Promise/, + "readArchiveEntryNative($1): Promise", + ) + .replace( + /sha256File\(([^)]*)\): Promise/, + "sha256File($1): Promise", + ) + .replace( + /copyFileRangeExclusive\(([^)]*)\): Promise/, + "copyFileRangeExclusive($1): Promise", + ); +if (hardenedTypes !== generatedTypes) { + writeFileSync(typesPath, hardenedTypes); + console.log(`hardened ${typesPath.slice(root.length)}`); +} diff --git a/scripts/native-loader.cjs b/scripts/native-loader.cjs new file mode 100644 index 0000000..687acf4 --- /dev/null +++ b/scripts/native-loader.cjs @@ -0,0 +1,179 @@ +// prettier-ignore +/* eslint-disable */ +// @ts-nocheck +/* Checked-in N-API loader for the seven platform packages this project ships. */ + +const { closeSync, openSync, readSync, readdirSync } = require('fs') +const loadErrors = [] + +const isFileMusl = (file) => file.includes('libc.musl-') || file.includes('ld-musl-') + +const isMuslFromReport = () => { + try { + if (!process.report || typeof process.report.getReport !== 'function') return null + const report = process.report.getReport() + if (report?.header?.glibcVersionRuntime) return false + if (report?.sharedObjects?.some(isFileMusl)) return true + } catch { + // Continue with filesystem and ELF inspection. + } + return null +} + +const isMuslFromFilesystem = () => { + for (const directory of ['/lib', '/usr/lib']) { + try { + if (readdirSync(directory).some(isFileMusl)) return true + } catch { + // A missing or unreadable conventional library directory is inconclusive. + } + } + return null +} + +const readUInt = (buffer, offset, bytes, littleEndian) => { + if (offset < 0 || offset + bytes > buffer.length) return null + if (bytes === 2) return littleEndian ? buffer.readUInt16LE(offset) : buffer.readUInt16BE(offset) + if (bytes === 4) return littleEndian ? buffer.readUInt32LE(offset) : buffer.readUInt32BE(offset) + if (bytes === 8) { + const value = littleEndian ? buffer.readBigUInt64LE(offset) : buffer.readBigUInt64BE(offset) + return value <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(value) : null + } + return null +} + +const isMuslFromElfInterpreter = () => { + let fd + try { + fd = openSync(process.execPath, 'r') + const header = Buffer.alloc(64) + if (readSync(fd, header, 0, header.length, 0) < 52) return null + if (!header.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) return null + + const elfClass = header[4] + const dataEncoding = header[5] + if ((elfClass !== 1 && elfClass !== 2) || (dataEncoding !== 1 && dataEncoding !== 2)) { + return null + } + const littleEndian = dataEncoding === 1 + const is64Bit = elfClass === 2 + const tableOffset = readUInt(header, is64Bit ? 32 : 28, is64Bit ? 8 : 4, littleEndian) + const entrySize = readUInt(header, is64Bit ? 54 : 42, 2, littleEndian) + const entryCount = readUInt(header, is64Bit ? 56 : 44, 2, littleEndian) + if (tableOffset === null || entrySize === null || entryCount === null || + entrySize < (is64Bit ? 56 : 32) || entryCount > 1024) return null + + const programHeader = Buffer.alloc(entrySize) + for (let index = 0; index < entryCount; index += 1) { + const offset = tableOffset + index * entrySize + if (readSync(fd, programHeader, 0, entrySize, offset) !== entrySize) return null + if (readUInt(programHeader, 0, 4, littleEndian) !== 3) continue // PT_INTERP + const interpreterOffset = readUInt(programHeader, is64Bit ? 8 : 4, is64Bit ? 8 : 4, littleEndian) + const interpreterSize = readUInt(programHeader, is64Bit ? 32 : 16, is64Bit ? 8 : 4, littleEndian) + if (interpreterOffset === null || interpreterSize === null || + interpreterSize < 1 || interpreterSize > 4096) return null + const interpreter = Buffer.alloc(interpreterSize) + if (readSync(fd, interpreter, 0, interpreterSize, interpreterOffset) !== interpreterSize) { + return null + } + return isFileMusl(interpreter.toString('utf8')) + } + } catch { + return null + } finally { + if (fd !== undefined) { + try { closeSync(fd) } catch { + // Best-effort detection must never turn close failure into import failure. + } + } + } + return null +} + +const isMusl = () => { + if (process.platform !== 'linux') return false + for (const detector of [isMuslFromReport, isMuslFromFilesystem, isMuslFromElfInterpreter]) { + const result = detector() + if (result !== null) return result + } + // Unknown Linux libc: try the glibc package and let its require fail normally. + return false +} + +const targetSuffix = () => { + if (process.platform === 'win32' && process.arch === 'x64') return 'win32-x64-msvc' + if (process.platform === 'darwin' && process.arch === 'x64') return 'darwin-x64' + if (process.platform === 'darwin' && process.arch === 'arm64') return 'darwin-arm64' + if (process.platform === 'linux' && (process.arch === 'x64' || process.arch === 'arm64')) { + return `linux-${process.arch}-${isMusl() ? 'musl' : 'gnu'}` + } + return null +} + +const tryRequire = (specifier) => { + try { + return require(specifier) + } catch (error) { + loadErrors.push(error) + return null + } +} + +const requireNative = () => { + if (process.env.NAPI_RS_NATIVE_LIBRARY_PATH) { + const explicit = tryRequire(process.env.NAPI_RS_NATIVE_LIBRARY_PATH) + if (explicit) return explicit + } + + const suffix = targetSuffix() + if (!suffix) { + loadErrors.push(new Error(`Unsupported OS or architecture: ${process.platform}-${process.arch}`)) + return null + } + const local = tryRequire(`./fs-safe-native.${suffix}.node`) + if (local) return local + + const packageName = `@openclaw/fs-safe-native-${suffix}` + const binding = tryRequire(packageName) + if (!binding) return null + if (process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + const expectedVersion = require('./package.json').version + const bindingVersion = require(`${packageName}/package.json`).version + if (bindingVersion !== expectedVersion) { + loadErrors.push(new Error( + `Native binding package version mismatch, expected ${expectedVersion} but got ${bindingVersion}. ` + + 'Reinstall dependencies to fix this issue.', + )) + return null + } + } + return binding +} + +const nativeBinding = requireNative() +if (!nativeBinding) { + const error = new Error( + `Cannot find native binding for ${process.platform}-${process.arch}; ` + + 'install the matching optional @openclaw/fs-safe-native platform package.', + ) + error.cause = loadErrors.reduceRight((cause, current) => { + current.cause = cause + return current + }, undefined) + throw error +} + +module.exports = nativeBinding +module.exports.cloneFileExclusive = nativeBinding.cloneFileExclusive +module.exports.copyFileRangeExclusive = nativeBinding.copyFileRangeExclusive +module.exports.createPrivateDirectory = nativeBinding.createPrivateDirectory +module.exports.extractArchiveNative = nativeBinding.extractArchiveNative +module.exports.fstatIdentity = nativeBinding.fstatIdentity +module.exports.inspectArchiveNative = nativeBinding.inspectArchiveNative +module.exports.linkBeneath = nativeBinding.linkBeneath +module.exports.mkdirBeneath = nativeBinding.mkdirBeneath +module.exports.openBeneath = nativeBinding.openBeneath +module.exports.readArchiveEntryNative = nativeBinding.readArchiveEntryNative +module.exports.readOwnerAndDacl = nativeBinding.readOwnerAndDacl +module.exports.renameNoReplace = nativeBinding.renameNoReplace +module.exports.sha256File = nativeBinding.sha256File diff --git a/scripts/native-mode-smoke.mjs b/scripts/native-mode-smoke.mjs new file mode 100644 index 0000000..9ebcb88 --- /dev/null +++ b/scripts/native-mode-smoke.mjs @@ -0,0 +1,29 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { createRequire } from "node:module"; +import { configureFsSafeNative, root } from "../dist/index.js"; + +const mode = process.argv[2]; +if (!new Set(["auto", "require", "off"]).has(mode)) { + throw new Error("usage: native-mode-smoke.mjs "); +} +configureFsSafeNative({ mode }); +if (mode !== "off") { + const require = createRequire(import.meta.url); + const binding = require("../native"); + if (typeof binding.openBeneath !== "function") throw new Error("native binding did not load"); +} else { + process.env.NAPI_RS_NATIVE_LIBRARY_PATH = path.join(os.tmpdir(), "must-not-load.node"); +} + +const directory = await fs.mkdtemp(path.join(os.tmpdir(), `fs-safe-mode-${mode}-`)); +try { + const capability = await root(directory); + await capability.create("value.txt", mode); + const value = await capability.readText("value.txt"); + if (value !== mode) throw new Error("mode smoke content mismatch"); + console.log(JSON.stringify({ mode, bindingExpected: mode !== "off", result: "PASS" })); +} finally { + await fs.rm(directory, { recursive: true, force: true }); +} diff --git a/scripts/native-smoke.mjs b/scripts/native-smoke.mjs new file mode 100644 index 0000000..15ed0a2 --- /dev/null +++ b/scripts/native-smoke.mjs @@ -0,0 +1,61 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { createRequire } from "node:module"; + +const require = createRequire(import.meta.url); +console.log("native smoke: load binding"); +const native = require("../native"); +const root = fs.mkdtempSync(path.join(os.tmpdir(), "fs-safe-native-smoke-")); +const rootFd = fs.openSync(root, fs.constants.O_RDONLY); +try { + console.log("native smoke: mkdir"); + try { + native.mkdirBeneath(rootFd, "nested", 0o700); + } catch (error) { + console.error("native smoke: mkdir failed", error); + throw error; + } + console.log("native smoke: write fixture"); + fs.writeFileSync(path.join(root, "nested", "source"), "source"); + console.log("native smoke: open"); + const fd = native.openBeneath(rootFd, "nested/source", fs.constants.O_RDONLY); + try { + console.log("native smoke: fstat"); + const identity = native.fstatIdentity(fd); + if (!identity.isFile || identity.size !== 6) throw new Error("unexpected native identity"); + } finally { + fs.closeSync(fd); + } + + console.log("native smoke: link"); + native.linkBeneath(rootFd, "nested/source", rootFd, "nested/linked"); + if (fs.readFileSync(path.join(root, "nested", "linked"), "utf8") !== "source") { + throw new Error("native hardlink did not preserve content"); + } + + console.log("native smoke: rename success and collision"); + native.renameNoReplace(rootFd, "nested/source", rootFd, "nested/renamed"); + fs.writeFileSync(path.join(root, "nested", "collision"), "collision"); + try { + native.renameNoReplace(rootFd, "nested/renamed", rootFd, "nested/collision"); + throw new Error("native collision unexpectedly succeeded"); + } catch (error) { + if (error.code !== "EEXIST") throw error; + } + + if (process.platform === "win32") { + console.log("native smoke: reject reparse component"); + fs.symlinkSync(path.join(root, "nested"), path.join(root, "alias"), "junction"); + try { + native.openBeneath(rootFd, "alias/linked", fs.constants.O_RDONLY); + throw new Error("native reparse traversal unexpectedly succeeded"); + } catch (error) { + if (error.message.includes("unexpectedly succeeded")) throw error; + } + } + console.log("native smoke: PASS"); +} finally { + fs.closeSync(rootFd); + fs.rmSync(root, { recursive: true, force: true }); +} diff --git a/scripts/packed-final-smoke.mjs b/scripts/packed-final-smoke.mjs new file mode 100644 index 0000000..dd3d733 --- /dev/null +++ b/scripts/packed-final-smoke.mjs @@ -0,0 +1,283 @@ +import assert from "node:assert/strict"; +import fsSync from "node:fs"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { extractArchive } from "@openclaw/fs-safe/archive"; +import { movePathWithCopyFallback } from "@openclaw/fs-safe/atomic"; +import { configureFsSafeNative, root as createRoot } from "@openclaw/fs-safe"; +import { publishFileExclusive, sha256File } from "@openclaw/fs-safe/durability"; +import { readOwnerAndDacl } from "@openclaw/fs-safe/permissions"; +import { __setFsSafeTestHooksForTest } from "@openclaw/fs-safe/test-hooks"; + +process.env.NODE_ENV = "test"; +configureFsSafeNative({ mode: "off" }); + +const root = await fs.mkdtemp(path.join(os.tmpdir(), "fs-safe-final-smoke-")); +const results = { archive: null, dacl: null, hash: null, moveHardlinks: null, publication: [], walk: null }; + +try { + const hashPath = path.join(root, "hash-input"); + await fs.writeFile(hashPath, "abc"); + results.hash = await sha256File(hashPath); + assert.deepEqual(results.hash, { + bytes: 3, + digest: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + }); + + if (process.platform === "win32") { + assert.throws( + () => readOwnerAndDacl(root), + (error) => error?.code === "helper-unavailable", + ); + results.dacl = { mode: "off", errorCode: "helper-unavailable" }; + } else { + results.dacl = readOwnerAndDacl(root); + assert.deepEqual(results.dacl, { + status: "unsupported-platform", + platform: process.platform, + }); + } + + const moveSource = path.join(root, "hardlinked-source"); + const moveAlias = path.join(root, "hardlinked-alias"); + const moveTarget = path.join(root, "hardlinked-target"); + await fs.writeFile(moveSource, "hardlinked"); + await fs.link(moveSource, moveAlias); + await assert.rejects( + movePathWithCopyFallback({ + from: moveSource, + to: moveTarget, + sourceHardlinks: "reject", + }), + (error) => error?.code === "hardlink", + ); + assert.equal(await pathExists(moveSource), true); + assert.equal(await pathExists(moveTarget), false); + results.moveHardlinks = { sameFilesystemRejected: true, errorCode: "hardlink" }; + + const archivePath = path.join(root, "fleet-restore.tar"); + const destination = path.join(root, "restore"); + await fs.writeFile(archivePath, symlinkTar("fleet/link", "../outside")); + await fs.mkdir(destination); + const started = performance.now(); + await assert.rejects( + settleWithin( + extractArchive({ + archivePath, + destDir: destination, + timeoutMs: 10_000, + entryFilter: (entry) => (entry.kind === "symlink" ? "skip" : "extract"), + }), + ), + (error) => error?.name === "ArchiveSecurityError" && error?.code === "entry-filtered", + ); + results.archive = { + mode: "off", + errorCode: "entry-filtered", + settledMs: Number((performance.now() - started).toFixed(2)), + }; + + const deepArchivePath = path.join(root, "deep-path.tar"); + const deepDestination = path.join(root, "deep-restore"); + await fs.writeFile( + deepArchivePath, + regularTar("one/two/three/four/value.txt", "payload"), + ); + await fs.mkdir(deepDestination); + await assert.rejects( + extractArchive({ + archivePath: deepArchivePath, + destDir: deepDestination, + timeoutMs: 10_000, + limits: { maxEntryPathComponents: 4 }, + }), + (error) => error?.code === "archive-entry-path-components-exceeds-limit", + ); + results.archive.pathLimitCode = "archive-entry-path-components-exceeds-limit"; + + const walkDirectory = path.join(root, "walk"); + await fs.mkdir(path.join(walkDirectory, "healthy"), { recursive: true }); + await fs.mkdir(path.join(walkDirectory, "pruned")); + await fs.mkdir(path.join(walkDirectory, "failed")); + await fs.writeFile(path.join(walkDirectory, "healthy", "value.txt"), "healthy"); + await fs.writeFile(path.join(walkDirectory, "pruned", "hidden.txt"), "hidden"); + const capability = await createRoot(walkDirectory); + const walked = []; + for await (const entry of capability.walk("", { + symlinkPolicy: "skip", + onDirectoryError: "skip-and-report", + entryFilter(entry) { + if (entry.relativePath === "pruned") return "skip-subtree"; + if (entry.relativePath === "failed") { + fsSync.rmSync(path.join(walkDirectory, "failed"), { recursive: true }); + } + return "include"; + }, + })) { + walked.push({ relativePath: entry.relativePath, kind: entry.kind }); + } + assert(walked.some((entry) => entry.relativePath === "healthy/value.txt")); + assert(!walked.some((entry) => entry.relativePath.startsWith("pruned"))); + assert(walked.some((entry) => entry.relativePath === "failed" && entry.kind === "directory-error")); + results.walk = { + prunedSubtree: true, + directoryErrorReported: true, + healthyEntryPreserved: true, + }; + + for (const cleanup of ["removed", "preserved", "unknown"]) { + const directory = path.join(root, `publish-${cleanup}`); + const sourcePath = path.join(directory, "source"); + const targetPath = path.join(directory, "target"); + const originalLinkPath = path.join(directory, "original-link"); + await fs.mkdir(directory); + await fs.writeFile(sourcePath, "source-content"); + __setFsSafeTestHooksForTest({ + async afterPublishTargetCreated(method) { + assert.equal(method, "hardlink"); + if (cleanup === "preserved") { + await fs.rename(targetPath, originalLinkPath); + await fs.writeFile(targetPath, "replacement-content"); + } + throw new Error("post-publication guard failed"); + }, + }); + const originalRm = fs.rm; + if (cleanup === "unknown") { + fs.rm = async () => { + throw Object.assign(new Error("cleanup denied"), { code: "EACCES" }); + }; + } + try { + await assert.rejects( + publishFileExclusive({ sourcePath, targetPath, strategy: "link-required" }), + (error) => + error?.name === "FsSafeError" && + error?.details?.phase === "hardlink-verify" && + error?.details?.targetCreated === true && + error?.details?.targetIdentity?.ino !== undefined && + error?.details?.cleanup === cleanup, + ); + } finally { + fs.rm = originalRm; + __setFsSafeTestHooksForTest(); + } + results.publication.push({ cleanup, phase: "hardlink-verify", targetCreated: true }); + } + + for (const onSyncFailure of ["rollback", "preserve"]) { + const directory = path.join(root, `publish-sync-${onSyncFailure}`); + const sourcePath = path.join(directory, "source"); + const targetPath = path.join(directory, "target"); + await fs.mkdir(directory); + await fs.writeFile(sourcePath, "complete-archive"); + __setFsSafeTestHooksForTest({ + beforePublishDirectorySync() { + throw Object.assign(new Error("directory sync failed"), { code: "EIO" }); + }, + }); + const expectedCleanup = onSyncFailure === "preserve" ? "preserved" : "removed"; + try { + await assert.rejects( + publishFileExclusive({ + sourcePath, + targetPath, + strategy: "link-required", + onSyncFailure, + }), + (error) => + error?.details?.phase === "directory-sync" && + error?.details?.targetCreated === true && + error?.details?.cleanup === expectedCleanup && + error?.details?.directorySync?.status === "failed" && + error?.details?.directorySync?.code === "EIO", + ); + } finally { + __setFsSafeTestHooksForTest(); + } + assert.equal(await pathExists(targetPath), onSyncFailure === "preserve"); + results.publication.push({ + onSyncFailure, + cleanup: expectedCleanup, + directorySync: { status: "failed", code: "EIO" }, + }); + } + + console.log(JSON.stringify(results)); +} finally { + __setFsSafeTestHooksForTest(); + await fs.rm(root, { recursive: true, force: true }); +} + +async function settleWithin(promise, milliseconds = 2_000) { + let timeout; + try { + return await Promise.race([ + promise, + new Promise((_resolve, reject) => { + timeout = setTimeout( + () => reject(new Error(`archive rejection did not settle within ${milliseconds}ms`)), + milliseconds, + ); + }), + ]); + } finally { + if (timeout) clearTimeout(timeout); + } +} + +function symlinkTar(entryPath, linkPath) { + const header = Buffer.alloc(512); + writeString(header, 0, 100, entryPath); + writeOctal(header, 100, 8, 0o777); + writeOctal(header, 108, 8, 0); + writeOctal(header, 116, 8, 0); + writeOctal(header, 124, 12, 0); + writeOctal(header, 136, 12, 0); + header.fill(0x20, 148, 156); + writeString(header, 156, 1, "2"); + writeString(header, 157, 100, linkPath); + writeString(header, 257, 6, "ustar\0"); + writeString(header, 263, 2, "00"); + const checksum = header.reduce((sum, byte) => sum + byte, 0); + writeString(header, 148, 8, `${checksum.toString(8).padStart(6, "0")}\0 `); + return Buffer.concat([header, Buffer.alloc(1024)]); +} + +function regularTar(entryPath, bodyText) { + const body = Buffer.from(bodyText); + const header = Buffer.alloc(512); + writeString(header, 0, 100, entryPath); + writeOctal(header, 100, 8, 0o644); + writeOctal(header, 108, 8, 0); + writeOctal(header, 116, 8, 0); + writeOctal(header, 124, 12, body.length); + writeOctal(header, 136, 12, 0); + header.fill(0x20, 148, 156); + writeString(header, 156, 1, "0"); + writeString(header, 257, 6, "ustar\0"); + writeString(header, 263, 2, "00"); + const checksum = header.reduce((sum, byte) => sum + byte, 0); + writeString(header, 148, 8, `${checksum.toString(8).padStart(6, "0")}\0 `); + const padding = Buffer.alloc((512 - (body.length % 512)) % 512); + return Buffer.concat([header, body, padding, Buffer.alloc(1024)]); +} + +function writeOctal(block, offset, length, value) { + writeString(block, offset, length, `${value.toString(8).padStart(length - 1, "0")}\0`); +} + +function writeString(block, offset, length, value) { + block.write(value, offset, Math.min(length, Buffer.byteLength(value)), "utf8"); +} + +async function pathExists(filePath) { + try { + await fs.lstat(filePath); + return true; + } catch (error) { + if (error?.code === "ENOENT") return false; + throw error; + } +} diff --git a/scripts/publish-or-verify.mjs b/scripts/publish-or-verify.mjs new file mode 100644 index 0000000..46bb777 --- /dev/null +++ b/scripts/publish-or-verify.mjs @@ -0,0 +1,59 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; + +const packageIndex = process.argv.indexOf("--package"); +const artifactsIndex = process.argv.indexOf("--artifacts"); +const packageName = packageIndex >= 0 ? process.argv[packageIndex + 1] : undefined; +const artifactsDir = resolve(artifactsIndex >= 0 ? process.argv[artifactsIndex + 1] : "release-artifacts"); +if (!packageName) throw new Error("--package is required"); + +const manifest = JSON.parse(readFileSync(join(artifactsDir, "manifest.json"), "utf8")); +const artifact = manifest.find((entry) => entry.name === packageName); +if (!artifact) throw new Error(`release manifest has no entry for ${packageName}`); +const spec = `${artifact.name}@${artifact.version}`; + +function registryState() { + try { + const raw = execFileSync("npm", ["view", spec, "dist", "--json"], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + const dist = JSON.parse(raw); + if (dist.integrity !== artifact.integrity) { + throw new Error(`${spec} exists with different package bytes`); + } + if (!dist.attestations?.url) { + throw new Error(`${spec} exists without npm provenance`); + } + return "verified"; + } catch (error) { + const stderr = String(error?.stderr ?? ""); + if (stderr.includes("E404")) return "missing"; + throw error; + } +} + +if (registryState() === "verified") { + console.log(`verified ${spec} integrity and provenance`); + process.exit(0); +} + +const published = spawnSync( + "npm", + ["publish", join(artifactsDir, artifact.filename), "--access", "public", "--provenance"], + { stdio: "inherit" }, +); +for (let attempt = 1; attempt <= 12; attempt += 1) { + try { + if (registryState() === "verified") { + console.log(`verified ${spec} integrity and provenance`); + process.exit(0); + } + } catch (error) { + if (attempt === 12) throw error; + } + console.log(`registry verification attempt ${attempt}/12 has not confirmed ${spec}`); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5_000); +} +throw new Error(`npm publish exited ${published.status}; registry never confirmed ${spec}`); diff --git a/scripts/release-graph-smoke.mjs b/scripts/release-graph-smoke.mjs new file mode 100644 index 0000000..59f32cd --- /dev/null +++ b/scripts/release-graph-smoke.mjs @@ -0,0 +1,241 @@ +import { execFileSync } from "node:child_process"; +import { + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, join, resolve } from "node:path"; + +const repositoryRoot = resolve(import.meta.dirname, ".."); +const proofPath = resolve(repositoryRoot, process.env.FS_SAFE_RELEASE_GRAPH_PROOF ?? "release-graph-proof.json"); +const workspace = mkdtempSync(join(tmpdir(), "fs-safe-release-graph-")); +const packsDirectory = join(workspace, "packs"); +const scratchDirectory = join(workspace, "consumer"); +const pnpmCli = process.env.npm_execpath; + +if (!pnpmCli || !existsSync(pnpmCli) || !basename(pnpmCli).startsWith("pnpm")) { + throw new Error("release graph smoke must run through pnpm"); +} + +const npmCli = resolveNpmCli(); +const cleanEnvironment = Object.fromEntries( + Object.entries(process.env).filter(([key]) => !key.toLowerCase().startsWith("npm_config_")), +); + +const host = resolveHostPackage(); +const nativeBinary = join(repositoryRoot, "native", host.binary); +const packageBinary = join(repositoryRoot, host.directory, host.binary); +const previousPackageBinary = existsSync(packageBinary) ? readFileSync(packageBinary) : undefined; + +try { + mkdirSync(packsDirectory); + mkdirSync(scratchDirectory); + if (!existsSync(nativeBinary)) { + throw new Error(`host native binary was not built: native/${host.binary}`); + } + copyFileSync(nativeBinary, packageBinary); + + const packageDirectories = [ + ...readdirSync(join(repositoryRoot, "npm"), { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => `npm/${entry.name}`) + .toSorted(), + "native", + ".", + ]; + const tarballs = new Map(); + for (const directory of packageDirectories) { + const packageRoot = resolve(repositoryRoot, directory); + const packageJson = JSON.parse(readFileSync(join(packageRoot, "package.json"), "utf8")); + const before = new Set(existsSync(packsDirectory) ? readdirSync(packsDirectory) : []); + runPnpm(["--dir", packageRoot, "pack", "--pack-destination", packsDirectory]); + const filename = readdirSync(packsDirectory).find((candidate) => !before.has(candidate)); + if (!filename) { + throw new Error(`pnpm pack did not produce an artifact for ${packageJson.name}`); + } + tarballs.set(packageJson.name, join(packsDirectory, filename)); + } + + const dependencies = Object.fromEntries( + ["@openclaw/fs-safe", "@openclaw/fs-safe-native", host.packageName].map((name) => [ + name, + `file:${tarballs.get(name)}`, + ]), + ); + writeFileSync( + join(scratchDirectory, "package.json"), + `${JSON.stringify({ private: true, type: "module", dependencies }, null, 2)}\n`, + ); + runNpm(["install", "--force", "--ignore-scripts", "--no-audit", "--no-fund", "--no-package-lock"], { + cwd: scratchDirectory, + }); + copyFileSync( + join(repositoryRoot, "scripts", "packed-final-smoke.mjs"), + join(scratchDirectory, "packed-final-smoke.mjs"), + ); + + const nativeOutput = runNode( + `import fs from 'node:fs';import os from 'node:os';import path from 'node:path';` + + `import {configureFsSafeNative} from '@openclaw/fs-safe';` + + `import {sha256File} from '@openclaw/fs-safe/durability';` + + `import {readOwnerAndDacl} from '@openclaw/fs-safe/permissions';` + + `const imported=await import('@openclaw/fs-safe-native');const n=imported.default??imported;` + + `if(typeof n.openBeneath!=='function')throw new Error('openBeneath missing');` + + `configureFsSafeNative({mode:'require'});const d=fs.mkdtempSync(path.join(os.tmpdir(),'fs-safe-hash-'));` + + `try{const p=path.join(d,'input');fs.writeFileSync(p,'abc');const h=await sha256File(p);` + + `if(h.digest!=='ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad')` + + `throw new Error('native hash mismatch');` + + `const a=readOwnerAndDacl(d);` + + `if(process.platform==='win32'&&(a.status!=='supported'||!/^s-/i.test(a.currentUserSid)))` + + `throw new Error('current process user SID missing');` + + `console.log(JSON.stringify({nativeBinding:'loaded',openBeneath:typeof n.openBeneath,publicHash:h,publicDacl:a}));}` + + `finally{fs.rmSync(d,{recursive:true,force:true});}`, + ); + const offOutput = runNode( + `import fs from 'node:fs';import os from 'node:os';import path from 'node:path';` + + `import {configureFsSafeNative} from '@openclaw/fs-safe';` + + `import {publishFileExclusive} from '@openclaw/fs-safe/durability';` + + `configureFsSafeNative({mode:'off'});` + + `const d=fs.mkdtempSync(path.join(os.tmpdir(),'fs-safe-off-'));` + + `try{const s=path.join(d,'source');const t=path.join(d,'target');fs.writeFileSync(s,'fallback');` + + `const r=await publishFileExclusive({sourcePath:s,targetPath:t,strategy:'link-or-copy'});` + + `if(fs.readFileSync(t,'utf8')!=='fallback')throw new Error('fallback content mismatch');` + + `console.log(JSON.stringify({nativeMode:'off',fallback:r.method}));}` + + `finally{fs.rmSync(d,{recursive:true,force:true});}`, + { NAPI_RS_NATIVE_LIBRARY_PATH: join(workspace, "must-not-load.node") }, + ); + const finalRoundOutput = runNodeFile("packed-final-smoke.mjs"); + + rmSync(join(scratchDirectory, "node_modules", "@openclaw", "fs-safe-native"), { + force: true, + recursive: true, + }); + rmSync(join(scratchDirectory, "node_modules", "@openclaw", host.packageName.slice("@openclaw/".length)), { + force: true, + recursive: true, + }); + const requireOutput = runNode( + `import fs from 'node:fs';import os from 'node:os';import path from 'node:path';` + + `import {configureFsSafeNative} from '@openclaw/fs-safe';` + + `import {publishFileExclusive} from '@openclaw/fs-safe/durability';` + + `configureFsSafeNative({mode:'require'});` + + `const d=fs.mkdtempSync(path.join(os.tmpdir(),'fs-safe-require-'));` + + `try{const s=path.join(d,'source');fs.writeFileSync(s,'require');` + + `try{await publishFileExclusive({sourcePath:s,targetPath:path.join(d,'target'),strategy:'rename-noreplace'});` + + `throw new Error('required native binding unexpectedly available');}` + + `catch(e){if(e.code!=='helper-unavailable')throw e;` + + `console.log(JSON.stringify({nativeMode:'require',bindingRemoved:true,errorCode:e.code}));}}` + + `finally{fs.rmSync(d,{recursive:true,force:true});}`, + ); + + const proof = { + schemaVersion: 1, + platform: `${process.platform}-${process.arch}`, + packagesPacked: packageDirectories.length, + hostPackage: host.packageName, + native: JSON.parse(nativeOutput), + fallback: JSON.parse(offOutput), + finalRound: JSON.parse(finalRoundOutput), + missingRequiredBinding: JSON.parse(requireOutput), + }; + writeFileSync(proofPath, `${JSON.stringify(proof, null, 2)}\n`); + console.log("release graph smoke: PASS"); + console.log(JSON.stringify(proof, null, 2)); +} finally { + if (previousPackageBinary) { + writeFileSync(packageBinary, previousPackageBinary); + } else { + rmSync(packageBinary, { force: true }); + } + rmSync(workspace, { force: true, recursive: true }); +} + +function resolveHostPackage() { + const key = `${process.platform}-${process.arch}`; + const hosts = { + "darwin-arm64": { + directory: "npm/darwin-arm64", + packageName: "@openclaw/fs-safe-native-darwin-arm64", + binary: "fs-safe-native.darwin-arm64.node", + }, + "darwin-x64": { + directory: "npm/darwin-x64", + packageName: "@openclaw/fs-safe-native-darwin-x64", + binary: "fs-safe-native.darwin-x64.node", + }, + "linux-arm64": { + directory: "npm/linux-arm64-gnu", + packageName: "@openclaw/fs-safe-native-linux-arm64-gnu", + binary: "fs-safe-native.linux-arm64-gnu.node", + }, + "linux-x64": { + directory: "npm/linux-x64-gnu", + packageName: "@openclaw/fs-safe-native-linux-x64-gnu", + binary: "fs-safe-native.linux-x64-gnu.node", + }, + "win32-x64": { + directory: "npm/win32-x64-msvc", + packageName: "@openclaw/fs-safe-native-win32-x64-msvc", + binary: "fs-safe-native.win32-x64-msvc.node", + }, + }; + const hostPackage = hosts[key]; + if (!hostPackage) { + throw new Error(`release graph smoke does not support ${key}`); + } + return hostPackage; +} + +function runPnpm(args) { + return execFileSync(process.execPath, [pnpmCli, ...args], { + cwd: repositoryRoot, + encoding: "utf8", + env: cleanEnvironment, + stdio: "pipe", + }); +} + +function runNpm(args, options) { + return execFileSync(process.execPath, [npmCli, ...args], { + ...options, + encoding: "utf8", + env: cleanEnvironment, + stdio: "pipe", + }); +} + +function runNode(source, additionalEnvironment = {}) { + return execFileSync(process.execPath, ["--input-type=module", "--eval", source], { + cwd: scratchDirectory, + encoding: "utf8", + env: { ...cleanEnvironment, ...additionalEnvironment }, + stdio: "pipe", + }).trim(); +} + +function runNodeFile(filename) { + return execFileSync(process.execPath, [filename], { + cwd: scratchDirectory, + encoding: "utf8", + env: cleanEnvironment, + stdio: "pipe", + }).trim(); +} + +function resolveNpmCli() { + const candidates = [ + join(dirname(process.execPath), "..", "lib", "node_modules", "npm", "bin", "npm-cli.js"), + join(dirname(process.execPath), "node_modules", "npm", "bin", "npm-cli.js"), + ]; + const npm = candidates.find((candidate) => existsSync(candidate)); + if (!npm) { + throw new Error("could not resolve npm-cli.js from the current Node installation"); + } + return npm; +} diff --git a/src/archive-entry.ts b/src/archive-entry.ts index 38a3051..0f64f91 100644 --- a/src/archive-entry.ts +++ b/src/archive-entry.ts @@ -1,4 +1,5 @@ import path from "node:path"; +import { ArchiveSecurityError } from "./archive-errors.js"; import { resolveSafeBaseDir } from "./path.js"; export function isWindowsDrivePath(value: string): boolean { @@ -17,15 +18,18 @@ export function validateArchiveEntryPath( return; } if (isWindowsDrivePath(entryPath)) { - throw new Error(`archive entry uses a drive path: ${entryPath}`); + throw new ArchiveSecurityError("entry-path", `archive entry uses a drive path: ${entryPath}`); } const normalized = path.posix.normalize(normalizeArchiveEntryPath(entryPath)); const escapeLabel = params?.escapeLabel ?? "destination"; if (normalized === ".." || normalized.startsWith("../")) { - throw new Error(`archive entry escapes ${escapeLabel}: ${entryPath}`); + throw new ArchiveSecurityError( + "entry-path", + `archive entry escapes ${escapeLabel}: ${entryPath}`, + ); } if (path.posix.isAbsolute(normalized) || normalized.startsWith("//")) { - throw new Error(`archive entry is absolute: ${entryPath}`); + throw new ArchiveSecurityError("entry-path", `archive entry is absolute: ${entryPath}`); } } @@ -55,7 +59,10 @@ export function resolveArchiveOutputPath(params: { const outPath = path.resolve(params.rootDir, params.relPath); const escapeLabel = params.escapeLabel ?? "destination"; if (!outPath.startsWith(safeBase)) { - throw new Error(`archive entry escapes ${escapeLabel}: ${params.originalPath}`); + throw new ArchiveSecurityError( + "entry-path", + `archive entry escapes ${escapeLabel}: ${params.originalPath}`, + ); } return outPath; } diff --git a/src/archive-errors.ts b/src/archive-errors.ts new file mode 100644 index 0000000..0e84019 --- /dev/null +++ b/src/archive-errors.ts @@ -0,0 +1,29 @@ +export type ArchiveFormatErrorCode = "archive-header-invalid"; + +export type ArchiveSecurityErrorCode = + | "destination-not-directory" + | "destination-symlink" + | "destination-symlink-traversal" + | "entry-filtered" + | "entry-link" + | "entry-path"; + +export class ArchiveSecurityError extends Error { + readonly code: ArchiveSecurityErrorCode; + + constructor(code: ArchiveSecurityErrorCode, message: string, options?: ErrorOptions) { + super(message, options); + this.code = code; + this.name = "ArchiveSecurityError"; + } +} + +export class ArchiveFormatError extends Error { + readonly code: ArchiveFormatErrorCode; + + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = "ArchiveFormatError"; + this.code = "archive-header-invalid"; + } +} diff --git a/src/archive-input.ts b/src/archive-input.ts new file mode 100644 index 0000000..b16f944 --- /dev/null +++ b/src/archive-input.ts @@ -0,0 +1,87 @@ +import { constants as fsConstants } from "node:fs"; +import type { FileHandle } from "node:fs/promises"; +import fs from "node:fs/promises"; +import path from "node:path"; +import type { ExtractionDeadline } from "./archive-deadline.js"; +import { writeFileHandleFully } from "./archive-file-io.js"; +import { + ARCHIVE_LIMIT_ERROR_CODE, + ArchiveLimitError, + type ResolvedArchiveExtractLimits, +} from "./archive-limits.js"; +import { sameFileIdentity } from "./file-identity.js"; +import { tempFile } from "./temp-target.js"; + +export type StagedArchiveFile = { path: string; cleanup: () => Promise }; + +async function closeFileHandle(handle: FileHandle | undefined): Promise { + if (handle) await handle.close().catch(() => undefined); +} + +export async function stageArchiveFileForExtraction(params: { + archivePath: string; + limits: ResolvedArchiveExtractLimits; + deadline: ExtractionDeadline; +}): Promise { + params.deadline.check(); + const sourcePath = path.resolve(params.archivePath); + const initialStat = await fs.lstat(sourcePath); + if (initialStat.isSymbolicLink() || !initialStat.isFile()) { + throw new Error(`archive is not a regular file: ${params.archivePath}`); + } + if (initialStat.size > params.limits.maxArchiveBytes) { + throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.ARCHIVE_SIZE_EXCEEDS_LIMIT); + } + const noFollow = + process.platform !== "win32" && "O_NOFOLLOW" in fsConstants ? fsConstants.O_NOFOLLOW : 0; + const handle = await fs.open(sourcePath, fsConstants.O_RDONLY | noFollow); + let staged: StagedArchiveFile | undefined; + let output: FileHandle | undefined; + try { + staged = await tempFile({ + prefix: "fs-safe-archive-input", + fileName: path.basename(sourcePath), + }); + const openedStat = await handle.stat(); + const pathStat = await fs.lstat(sourcePath); + if ( + !openedStat.isFile() || + pathStat.isSymbolicLink() || + !pathStat.isFile() || + !sameFileIdentity(initialStat, openedStat) || + !sameFileIdentity(pathStat, openedStat) + ) { + throw new Error("archive changed during validation"); + } + + const flags = + fsConstants.O_WRONLY | + fsConstants.O_CREAT | + fsConstants.O_EXCL | + (process.platform !== "win32" && "O_NOFOLLOW" in fsConstants + ? fsConstants.O_NOFOLLOW + : 0); + output = await fs.open(staged.path, flags, 0o600); + const buffer = Buffer.allocUnsafe(64 * 1024); + let written = 0; + while (true) { + params.deadline.check(); + const { bytesRead } = await handle.read(buffer, 0, buffer.length, null); + if (bytesRead === 0) break; + written += bytesRead; + if (written > params.limits.maxArchiveBytes) { + throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.ARCHIVE_SIZE_EXCEEDS_LIMIT); + } + await writeFileHandleFully({ handle: output, buffer, bytes: bytesRead, deadline: params.deadline }); + } + await output.close(); + output = undefined; + return staged; + } catch (error) { + await closeFileHandle(output); + await staged?.cleanup().catch(() => undefined); + throw error; + } finally { + await closeFileHandle(handle); + } +} diff --git a/src/archive-kind.ts b/src/archive-kind.ts index 5804b41..bf8342b 100644 --- a/src/archive-kind.ts +++ b/src/archive-kind.ts @@ -1,16 +1,38 @@ import fs from "node:fs/promises"; import path from "node:path"; +import { FsSafeError } from "./errors.js"; +import { getNativeBinding } from "./native.js"; import { normalizeLowercaseStringOrEmpty } from "./string-coerce.js"; -export type ArchiveKind = "tar" | "zip"; +export type ArchiveKind = "tar" | "tar-bzip2" | "tar-zstd" | "zip"; const TAR_SUFFIXES = [".tgz", ".tar.gz", ".tar"]; +const NATIVE_TAR_SUFFIXES = [ + { suffixes: [".tbz2", ".tbz", ".tar.bz2"], kind: "tar-bzip2" }, + { suffixes: [".tzst", ".tar.zst", ".tar.zstd"], kind: "tar-zstd" }, +] as const; + +function requireNativeArchiveKind(kind: "tar-bzip2" | "tar-zstd"): ArchiveKind { + if (!getNativeBinding()) { + throw new FsSafeError( + "helper-unavailable", + `${kind} archives require the optional native binding; install the matching ` + + "@openclaw/fs-safe-native platform package and use FS_SAFE_NATIVE_MODE=auto or require", + ); + } + return kind; +} export function resolveArchiveKind(filePath: string): ArchiveKind | null { const lower = normalizeLowercaseStringOrEmpty(filePath); if (lower.endsWith(".zip")) { return "zip"; } + for (const { suffixes, kind } of NATIVE_TAR_SUFFIXES) { + if (suffixes.some((suffix) => lower.endsWith(suffix))) { + return requireNativeArchiveKind(kind); + } + } if (TAR_SUFFIXES.some((suffix) => lower.endsWith(suffix))) { return "tar"; } diff --git a/src/archive-limits.ts b/src/archive-limits.ts index 1b1f7ab..64e0b93 100644 --- a/src/archive-limits.ts +++ b/src/archive-limits.ts @@ -11,18 +11,27 @@ export type ArchiveExtractLimits = { maxExtractedBytes?: number; /** Max extracted bytes for a single file entry. */ maxEntryBytes?: number; + /** Max bytes in one PAX, GNU long-name, or related TAR metadata entry. */ + maxMetaEntryBytes?: number; + /** Max path components in one extracted entry after stripComponents. */ + maxEntryPathComponents?: number; }; export const DEFAULT_MAX_ARCHIVE_BYTES_ZIP = 256 * 1024 * 1024; export const DEFAULT_MAX_ENTRIES = 50_000; export const DEFAULT_MAX_EXTRACTED_BYTES = 512 * 1024 * 1024; export const DEFAULT_MAX_ENTRY_BYTES = 256 * 1024 * 1024; +export const DEFAULT_MAX_META_ENTRY_BYTES = 1024 * 1024; +export const DEFAULT_MAX_ENTRY_PATH_COMPONENTS = 256; export const ARCHIVE_LIMIT_ERROR_CODE = { ARCHIVE_SIZE_EXCEEDS_LIMIT: "archive-size-exceeds-limit", ENTRY_COUNT_EXCEEDS_LIMIT: "archive-entry-count-exceeds-limit", ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT: "archive-entry-extracted-size-exceeds-limit", EXTRACTED_SIZE_EXCEEDS_LIMIT: "archive-extracted-size-exceeds-limit", + META_ENTRY_SIZE_EXCEEDS_LIMIT: "archive-meta-entry-size-exceeds-limit", + MANIFEST_SIZE_EXCEEDS_LIMIT: "archive-manifest-size-exceeds-limit", + ENTRY_PATH_COMPONENTS_EXCEEDS_LIMIT: "archive-entry-path-components-exceeds-limit", } as const; export type ArchiveLimitErrorCode = @@ -34,6 +43,12 @@ const ARCHIVE_LIMIT_ERROR_MESSAGE = { [ARCHIVE_LIMIT_ERROR_CODE.ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT]: "archive entry extracted size exceeds limit", [ARCHIVE_LIMIT_ERROR_CODE.EXTRACTED_SIZE_EXCEEDS_LIMIT]: "archive extracted size exceeds limit", + [ARCHIVE_LIMIT_ERROR_CODE.META_ENTRY_SIZE_EXCEEDS_LIMIT]: + "archive metadata entry size exceeds limit", + [ARCHIVE_LIMIT_ERROR_CODE.MANIFEST_SIZE_EXCEEDS_LIMIT]: + "archive manifest size exceeds limit", + [ARCHIVE_LIMIT_ERROR_CODE.ENTRY_PATH_COMPONENTS_EXCEEDS_LIMIT]: + "archive entry path components exceed limit", } as const satisfies Record; export class ArchiveLimitError extends Error { @@ -65,9 +80,27 @@ export function resolveExtractLimits( maxEntries: clampLimit(limits?.maxEntries) ?? DEFAULT_MAX_ENTRIES, maxExtractedBytes: clampLimit(limits?.maxExtractedBytes) ?? DEFAULT_MAX_EXTRACTED_BYTES, maxEntryBytes: clampLimit(limits?.maxEntryBytes) ?? DEFAULT_MAX_ENTRY_BYTES, + maxMetaEntryBytes: + clampLimit(limits?.maxMetaEntryBytes) ?? DEFAULT_MAX_META_ENTRY_BYTES, + maxEntryPathComponents: + clampLimit(limits?.maxEntryPathComponents) ?? DEFAULT_MAX_ENTRY_PATH_COMPONENTS, }; } +export function assertArchiveEntryPathComponentsWithinLimit( + entryPath: string, + limits: ResolvedArchiveExtractLimits, +): void { + const components = entryPath + .split(/[\\/]+/u) + .filter((component) => component.length > 0 && component !== ".").length; + if (components > limits.maxEntryPathComponents) { + throw new ArchiveLimitError( + ARCHIVE_LIMIT_ERROR_CODE.ENTRY_PATH_COMPONENTS_EXCEEDS_LIMIT, + ); + } +} + export function assertArchiveEntryCountWithinLimit( entryCount: number, limits: ResolvedArchiveExtractLimits, diff --git a/src/archive-native.ts b/src/archive-native.ts new file mode 100644 index 0000000..c89e18c --- /dev/null +++ b/src/archive-native.ts @@ -0,0 +1,173 @@ +import { constants as fsConstants } from "node:fs"; +import fs from "node:fs/promises"; +import { ArchiveFormatError, ArchiveSecurityError } from "./archive-errors.js"; +import { resolveArchiveOutputPath, stripArchivePath, validateArchiveEntryPath } from "./archive-entry.js"; +import type { ExtractionDeadline } from "./archive-deadline.js"; +import { stageArchiveFileForExtraction } from "./archive-input.js"; +import type { ArchiveKind } from "./archive-kind.js"; +import { + ARCHIVE_LIMIT_ERROR_CODE, + ArchiveLimitError, + assertArchiveEntryCountWithinLimit, + assertArchiveEntryPathComponentsWithinLimit, + createByteBudgetTracker, + resolveExtractLimits, + type ArchiveExtractLimits, +} from "./archive-limits.js"; +import type { ExtractArchiveOptions } from "./archive-options.js"; +import { resolveArchiveEntryMode, shouldExtractArchiveEntry } from "./archive-policy.js"; +import { + mergeExtractedTreeIntoDestination, + prepareArchiveDestinationDir, + withStagedArchiveDestination, +} from "./archive-staging.js"; +import type { NativeBinding } from "./native.js"; + +function policyKind(kind: string): "file" | "directory" | "symlink" | "other" { + if (kind === "file" || kind === "directory") return kind; + if (kind === "symlink" || kind === "hardlink") return "symlink"; + return "other"; +} + +function throwMappedNativeError(error: unknown): never { + if (error instanceof Error) { + for (const code of Object.values(ARCHIVE_LIMIT_ERROR_CODE)) { + if (error.message.includes(code)) throw new ArchiveLimitError(code); + } + if (error.message.includes("archive-header-invalid")) { + throw new ArchiveFormatError(error.message, { cause: error }); + } + } + throw error; +} + +export async function extractNativeArchive(params: { + binding: NativeBinding; + archivePath: string; + destDir: string; + kind: ArchiveKind; + stripComponents?: number; + limits?: ArchiveExtractLimits; + deadline: ExtractionDeadline; + entryModes?: ExtractArchiveOptions["entryModes"]; + entryFilter?: ExtractArchiveOptions["entryFilter"]; + onFiltered?: ExtractArchiveOptions["onFiltered"]; +}): Promise { + const limits = resolveExtractLimits(params.limits); + const stagedArchive = await stageArchiveFileForExtraction({ + archivePath: params.archivePath, + limits, + deadline: params.deadline, + }); + try { + const destinationRealDir = await prepareArchiveDestinationDir(params.destDir); + await withStagedArchiveDestination({ + destinationRealDir, + run: async (stagingDir) => { + params.deadline.check(); + const manifest = await params.binding + .inspectArchiveNative( + stagedArchive.path, + params.kind, + limits.maxEntries, + limits.maxMetaEntryBytes, + limits.maxArchiveBytes, + params.deadline.signal, + ) + .catch(throwMappedNativeError); + params.deadline.check(); + assertArchiveEntryCountWithinLimit(manifest.length, limits); + const strip = Math.max(0, Math.floor(params.stripComponents ?? 0)); + const budget = createByteBudgetTracker(limits); + const plan: Array<{ + index: number; + path: string; + kind: string; + size: number; + mode: number; + }> = []; + + for (const entry of manifest) { + params.deadline.check(); + validateArchiveEntryPath(entry.path); + const relPath = stripArchivePath(entry.path, strip); + if (!relPath) continue; + validateArchiveEntryPath(relPath); + assertArchiveEntryPathComponentsWithinLimit(relPath, limits); + resolveArchiveOutputPath({ rootDir: stagingDir, relPath, originalPath: entry.path }); + const kind = policyKind(entry.kind); + if ( + !shouldExtractArchiveEntry({ + filter: params.entryFilter, + onFiltered: params.onFiltered, + entry: { path: entry.path, kind, size: entry.size }, + }) + ) { + continue; + } + if (entry.kind === "sparse") { + throw new ArchiveFormatError( + `GNU sparse archive entry is not supported: ${entry.path}`, + ); + } + if (kind === "symlink") { + const label = params.kind === "zip" ? "zip" : "tar"; + throw new ArchiveSecurityError( + "entry-link", + `${label} entry is a link: ${entry.path}`, + ); + } + if (!Number.isSafeInteger(entry.size) || entry.size < 0) { + throw new ArchiveLimitError( + ARCHIVE_LIMIT_ERROR_CODE.ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT, + ); + } + if (kind === "file") { + budget.startEntry(); + budget.addEntrySize(entry.size); + } + if (kind === "file" || kind === "directory") { + plan.push({ + index: entry.index, + path: relPath, + kind, + size: entry.size, + mode: resolveArchiveEntryMode({ + kind, + archivedMode: entry.mode, + policy: params.entryModes, + }), + }); + } + } + + const directory = await fs.open( + stagingDir, + fsConstants.O_RDONLY | + (typeof fsConstants.O_DIRECTORY === "number" ? fsConstants.O_DIRECTORY : 0), + ); + try { + params.deadline.check(); + await params.binding.extractArchiveNative( + stagedArchive.path, + params.kind, + directory.fd, + plan, + limits.maxMetaEntryBytes, + params.deadline.signal, + ); + } finally { + await directory.close().catch(() => undefined); + } + params.deadline.check(); + await mergeExtractedTreeIntoDestination({ + sourceDir: stagingDir, + destinationDir: params.destDir, + destinationRealDir, + }); + }, + }); + } finally { + await stagedArchive.cleanup(); + } +} diff --git a/src/archive-options.ts b/src/archive-options.ts new file mode 100644 index 0000000..594fbed --- /dev/null +++ b/src/archive-options.ts @@ -0,0 +1,26 @@ +import type { ArchiveKind } from "./archive-kind.js"; +import type { ArchiveExtractLimits } from "./archive-limits.js"; +import type { + ArchiveEntryFilter, + ArchiveEntryModePolicy, + ArchiveFilteredEntryPolicy, +} from "./archive-policy.js"; + +export type ArchiveLogger = { + info?: (message: string) => void; + warn?: (message: string) => void; +}; + +export type ExtractArchiveOptions = { + archivePath: string; + destDir: string; + timeoutMs: number; + kind?: ArchiveKind; + stripComponents?: number; + tarGzip?: boolean; + limits?: ArchiveExtractLimits; + logger?: ArchiveLogger; + entryModes?: ArchiveEntryModePolicy; + entryFilter?: ArchiveEntryFilter; + onFiltered?: ArchiveFilteredEntryPolicy; +}; diff --git a/src/archive-policy.ts b/src/archive-policy.ts new file mode 100644 index 0000000..fe57068 --- /dev/null +++ b/src/archive-policy.ts @@ -0,0 +1,49 @@ +import { ArchiveSecurityError } from "./archive-errors.js"; + +export type ArchiveEntryKind = "file" | "directory" | "symlink" | "other"; +export type ArchiveEntryModePolicy = "clamp" | "preserve"; +export type ArchiveFilteredEntryPolicy = "reject-archive" | "skip-entry"; +export type ArchiveEntryFilter = (entry: { + path: string; + kind: ArchiveEntryKind; + size: number; +}) => "extract" | "skip"; + +export function archiveEntryKindFromTarType(type: string): ArchiveEntryKind { + if (type === "Directory" || type === "GNUDumpDir") return "directory"; + if (type === "File" || type === "OldFile" || type === "ContiguousFile") return "file"; + if (type === "SymbolicLink" || type === "Link") return "symlink"; + return "other"; +} + +export function resolveArchiveEntryMode(params: { + kind: "file" | "directory"; + archivedMode?: number; + policy?: ArchiveEntryModePolicy; +}): number { + const archivedMode = (params.archivedMode ?? 0) & 0o777; + if (params.policy === "preserve") { + return archivedMode || (params.kind === "directory" ? 0o755 : 0o644); + } + if (params.kind === "directory") { + return 0o755; + } + return archivedMode & 0o100 ? 0o755 : 0o644; +} + +export function shouldExtractArchiveEntry(params: { + filter?: ArchiveEntryFilter; + onFiltered?: ArchiveFilteredEntryPolicy; + entry: Parameters[0]; +}): boolean { + if (!params.filter || params.filter(params.entry) === "extract") { + return true; + } + if ((params.onFiltered ?? "reject-archive") === "reject-archive") { + throw new ArchiveSecurityError( + "entry-filtered", + `archive entry rejected by filter: ${params.entry.path}`, + ); + } + return false; +} diff --git a/src/archive-read.ts b/src/archive-read.ts new file mode 100644 index 0000000..0187b14 --- /dev/null +++ b/src/archive-read.ts @@ -0,0 +1,264 @@ +import fsSync from "node:fs"; +import fs from "node:fs/promises"; +import { Readable } from "node:stream"; +import { readFileHandleBounded } from "./bounded-read.js"; +import { ArchiveFormatError } from "./archive-errors.js"; +import { + normalizeArchiveEntryPath, + validateArchiveEntryPath, +} from "./archive-entry.js"; +import { resolveArchiveKind, type ArchiveKind } from "./archive-kind.js"; +import { + DEFAULT_MAX_ARCHIVE_BYTES_ZIP, + DEFAULT_MAX_ENTRIES, + DEFAULT_MAX_META_ENTRY_BYTES, + ArchiveLimitError, + ARCHIVE_LIMIT_ERROR_CODE, +} from "./archive-limits.js"; +import { readTarEntryInfo } from "./archive-tar.js"; +import { preflightTarMetadata } from "./archive-tar-meta.js"; +import { importOptionalTar } from "./archive-tar-runtime.js"; +import { loadZipArchiveWithPreflight } from "./archive-zip-preflight.js"; +import { FsSafeError } from "./errors.js"; +import { sameFileIdentity } from "./file-identity.js"; +import { getNativeBinding } from "./native.js"; +import { tempFile } from "./temp-target.js"; + +type ZipEntry = { + name: string; + dir: boolean; + unixPermissions?: number; + nodeStream?: () => NodeJS.ReadableStream; + async(type: "nodebuffer"): Promise; +}; + +const ZIP_UNIX_FILE_TYPE_MASK = 0o170000; +const ZIP_UNIX_SYMLINK_TYPE = 0o120000; + +function normalizedRequestedEntry(entryPath: string): string { + validateArchiveEntryPath(entryPath, { escapeLabel: "archive root" }); + const normalized = normalizeArchiveEntryPath(entryPath).replace(/^\.\//, ""); + if (!normalized || normalized.endsWith("/")) { + throw new Error(`archive entry is not a file: ${entryPath}`); + } + return normalized; +} + +async function readStreamBounded( + stream: NodeJS.ReadableStream | AsyncIterable, + maxBytes: number, +): Promise { + if (!(Symbol.asyncIterator in Object(stream))) { + return await new Promise((resolve, reject) => { + const readable = stream as NodeJS.ReadableStream; + const chunks: Buffer[] = []; + let total = 0; + readable.on("data", (chunk: unknown) => { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array); + total += buffer.length; + if (total > maxBytes) { + readable.pause(); + reject( + new ArchiveLimitError( + ARCHIVE_LIMIT_ERROR_CODE.ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT, + ), + ); + return; + } + chunks.push(buffer); + }); + readable.once("end", () => resolve(Buffer.concat(chunks, total))); + readable.once("error", reject); + }); + } + const chunks: Buffer[] = []; + let total = 0; + for await (const chunk of stream as AsyncIterable) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array); + total += buffer.length; + if (total > maxBytes) { + throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT); + } + chunks.push(buffer); + } + return Buffer.concat(chunks, total); +} + +async function stageArchiveInput(archivePath: string): Promise<{ + path: string; + buffer: Buffer; + cleanup(): Promise; +}> { + const resolved = await fs.realpath(archivePath); + const before = await fs.lstat(archivePath); + if (before.isSymbolicLink() || !before.isFile()) { + throw new Error(`archive is not a regular file: ${archivePath}`); + } + const noFollow = + process.platform !== "win32" && typeof fsSync.constants.O_NOFOLLOW === "number" + ? fsSync.constants.O_NOFOLLOW + : 0; + const handle = await fs.open(resolved, fsSync.constants.O_RDONLY | noFollow); + const staged = await tempFile({ prefix: "fs-safe-archive-read", fileName: "archive.bin" }); + try { + const opened = await handle.stat(); + const current = await fs.lstat(resolved); + if ( + !opened.isFile() || + !current.isFile() || + !sameFileIdentity(before, opened) || + !sameFileIdentity(current, opened) + ) { + throw new Error("archive changed during validation"); + } + const buffer = await readFileHandleBounded(handle, DEFAULT_MAX_ARCHIVE_BYTES_ZIP); + await fs.writeFile(staged.path, buffer, { flag: "wx", mode: 0o600 }); + return { path: staged.path, buffer, cleanup: staged.cleanup }; + } catch (error) { + await staged.cleanup().catch(() => undefined); + throw error; + } finally { + await handle.close().catch(() => undefined); + } +} + +async function readZipEntry(buffer: Buffer, entryPath: string, maxBytes: number): Promise { + const archive = await loadZipArchiveWithPreflight(buffer, { + maxArchiveBytes: DEFAULT_MAX_ARCHIVE_BYTES_ZIP, + maxEntryBytes: maxBytes, + maxExtractedBytes: maxBytes, + }); + const entry = (archive.files as Record)[entryPath]; + if (!entry || entry.dir) { + throw new Error(`archive entry not found: ${entryPath}`); + } + if ( + typeof entry.unixPermissions === "number" && + (entry.unixPermissions & ZIP_UNIX_FILE_TYPE_MASK) === ZIP_UNIX_SYMLINK_TYPE + ) { + throw new Error(`archive entry is a link: ${entryPath}`); + } + const stream = + typeof entry.nodeStream === "function" + ? entry.nodeStream() + : Readable.from(await entry.async("nodebuffer")); + return await readStreamBounded(stream, maxBytes); +} + +async function readTarEntry(archivePath: string, entryPath: string, maxBytes: number): Promise { + const tar = await importOptionalTar(); + await preflightTarMetadata({ + archivePath, + maxMetaEntryBytes: DEFAULT_MAX_META_ENTRY_BYTES, + }); + let matched: Promise | undefined; + await tar.t({ + file: archivePath, + strict: true, + maxMetaEntrySize: DEFAULT_MAX_META_ENTRY_BYTES, + onReadEntry(entry) { + const info = readTarEntryInfo(entry); + validateArchiveEntryPath(info.path, { escapeLabel: "archive root" }); + const normalized = normalizeArchiveEntryPath(info.path).replace(/^\.\//, ""); + if (normalized !== entryPath) { + entry.resume(); + return; + } + if (info.type !== "File" && info.type !== "OldFile" && info.type !== "ContiguousFile") { + matched = Promise.reject(new Error(`archive entry is not a file: ${entryPath}`)); + entry.resume(); + return; + } + if (info.size > maxBytes) { + matched = Promise.reject( + new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT), + ); + entry.resume(); + return; + } + matched = readStreamBounded(entry, maxBytes); + }, + }); + if (!matched) { + throw new Error(`archive entry not found: ${entryPath}`); + } + return await matched; +} + +export async function readArchiveEntry( + archivePath: string, + entryPath: string, + options: { maxBytes: number; kind?: ArchiveKind }, +): Promise { + if (!Number.isSafeInteger(options.maxBytes) || options.maxBytes < 0) { + throw new RangeError("maxBytes must be a non-negative safe integer"); + } + const kind = options.kind ?? resolveArchiveKind(archivePath); + if (!kind) { + throw new Error(`unsupported archive: ${archivePath}`); + } + const requestedEntry = normalizedRequestedEntry(entryPath); + const staged = await stageArchiveInput(archivePath); + try { + const native = getNativeBinding(); + if (native) { + try { + const signal = new AbortController().signal; + const manifest = await native.inspectArchiveNative( + staged.path, + kind, + DEFAULT_MAX_ENTRIES, + DEFAULT_MAX_META_ENTRY_BYTES, + DEFAULT_MAX_ARCHIVE_BYTES_ZIP, + signal, + ); + let rawEntryPath: string | undefined; + for (const entry of manifest) { + validateArchiveEntryPath(entry.path, { escapeLabel: "archive root" }); + const normalized = normalizeArchiveEntryPath(entry.path).replace(/^\.\//, ""); + if (normalized === requestedEntry) { + if (entry.kind !== "file") { + throw new Error(`archive entry is not a file: ${entryPath}`); + } + rawEntryPath = entry.path; + break; + } + } + if (!rawEntryPath) throw new Error(`archive entry not found: ${entryPath}`); + return await native.readArchiveEntryNative( + staged.path, + kind, + rawEntryPath, + options.maxBytes, + DEFAULT_MAX_ENTRIES, + DEFAULT_MAX_META_ENTRY_BYTES, + signal, + ); + } catch (error) { + if ( + error instanceof Error && + error.message.includes(ARCHIVE_LIMIT_ERROR_CODE.ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT) + ) { + throw new ArchiveLimitError( + ARCHIVE_LIMIT_ERROR_CODE.ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT, + ); + } + if (error instanceof Error && error.message.includes("archive-header-invalid")) { + throw new ArchiveFormatError(error.message, { cause: error }); + } + throw error; + } + } + if (kind === "tar-zstd" || kind === "tar-bzip2") { + throw new FsSafeError( + "helper-unavailable", + `${kind} archives require the optional native binding; install the matching platform package`, + ); + } + return kind === "zip" + ? await readZipEntry(staged.buffer, requestedEntry, options.maxBytes) + : await readTarEntry(staged.path, requestedEntry, options.maxBytes); + } finally { + await staged.cleanup(); + } +} diff --git a/src/archive-staging.ts b/src/archive-staging.ts index 19be272..b9dac1d 100644 --- a/src/archive-staging.ts +++ b/src/archive-staging.ts @@ -6,6 +6,10 @@ import { createAsyncDirectoryGuard, type AsyncDirectoryGuard, } from "./directory-guard.js"; +import { + ArchiveSecurityError, + type ArchiveSecurityErrorCode, +} from "./archive-errors.js"; import { FsSafeError } from "./errors.js"; import { resolveOpenedFileRealPathForHandle, root } from "./root.js"; import { isNotFoundPathError, isPathInside } from "./path.js"; @@ -15,20 +19,7 @@ import { getFsSafeTestHooks } from "./test-hooks.js"; const ERROR_ARCHIVE_ENTRY_TRAVERSES_SYMLINK = "archive entry traverses symlink in destination"; const ARCHIVE_STAGING_MODE = 0o700; -export type ArchiveSecurityErrorCode = - | "destination-not-directory" - | "destination-symlink" - | "destination-symlink-traversal"; - -export class ArchiveSecurityError extends Error { - code: ArchiveSecurityErrorCode; - - constructor(code: ArchiveSecurityErrorCode, message: string, options?: ErrorOptions) { - super(message, options); - this.code = code; - this.name = "ArchiveSecurityError"; - } -} +export { ArchiveSecurityError, type ArchiveSecurityErrorCode } from "./archive-errors.js"; function symlinkTraversalError(originalPath: string): ArchiveSecurityError { return new ArchiveSecurityError( @@ -135,6 +126,21 @@ async function assertResolvedInsideDestination(params: { } } +async function mkdirArchiveOutput(params: { + targetRoot: { mkdir(relativePath: string): Promise }; + relativePath: string; + originalPath: string; +}): Promise { + try { + await params.targetRoot.mkdir(params.relativePath); + } catch (error) { + if (error instanceof FsSafeError) { + throw symlinkTraversalError(params.originalPath); + } + throw error; + } +} + export async function prepareArchiveOutputPath(params: { destinationDir: string; destinationRealDir: string; @@ -155,7 +161,7 @@ export async function prepareArchiveOutputPath(params: { if (params.isDirectory) { await getFsSafeTestHooks()?.beforeArchiveOutputMutation?.("mkdir", params.outPath); await assertDirectoryIdentityGuard(destinationGuard); - await targetRoot.mkdir(relPath); + await mkdirArchiveOutput({ targetRoot, relativePath: relPath, originalPath: params.originalPath }); await assertDirectoryIdentityGuard(destinationGuard); await assertResolvedInsideDestination({ destinationRealDir: params.destinationRealDir, @@ -169,7 +175,11 @@ export async function prepareArchiveOutputPath(params: { if (parentRel !== ".") { await getFsSafeTestHooks()?.beforeArchiveOutputMutation?.("mkdir", path.dirname(params.outPath)); await assertDirectoryIdentityGuard(destinationGuard); - await targetRoot.mkdir(parentRel); + await mkdirArchiveOutput({ + targetRoot, + relativePath: parentRel, + originalPath: params.originalPath, + }); await assertDirectoryIdentityGuard(destinationGuard); } await assertResolvedInsideDestination({ diff --git a/src/archive-tar-meta.ts b/src/archive-tar-meta.ts new file mode 100644 index 0000000..3831d14 --- /dev/null +++ b/src/archive-tar-meta.ts @@ -0,0 +1,149 @@ +import fs from "node:fs"; +import { Transform, Writable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { createGunzip } from "node:zlib"; +import { ArchiveFormatError } from "./archive-errors.js"; +import { ARCHIVE_LIMIT_ERROR_CODE, ArchiveLimitError } from "./archive-limits.js"; + +type MeterState = + | { kind: "header" } + | { kind: "data"; remaining: number } + | { kind: "sparse"; dataRemaining: number; metaBytes: number }; + +class TarMetadataMeter extends Transform { + private readonly block = Buffer.alloc(512); + private blockLength = 0; + private state: MeterState = { kind: "header" }; + + constructor(private readonly maxMetaEntryBytes: number) { + super(); + } + + private invalid(message: string): ArchiveFormatError { + return new ArchiveFormatError(`invalid TAR header: ${message}`); + } + + private parseSize(): number { + const field = this.block.subarray(124, 136); + if ((field[0] ?? 0) & 0x80) { + const value = field.readBigUInt64BE(4); + if (value > BigInt(Number.MAX_SAFE_INTEGER)) { + throw this.invalid("base-256 size exceeds the safe integer range"); + } + return Number(value); + } + const zero = field.indexOf(0); + const text = field.subarray(0, zero < 0 ? field.length : zero).toString("ascii").trim(); + if (!text || !/^[0-7]+$/.test(text)) throw this.invalid("size is not valid octal"); + const value = Number.parseInt(text, 8); + if (!Number.isSafeInteger(value)) throw this.invalid("octal size exceeds the safe integer range"); + return value; + } + + private finishHeader(): void { + if (this.block.every((byte) => byte === 0)) { + this.blockLength = 0; + this.state = { kind: "header" }; + return; + } + const size = this.parseSize(); + const type = this.block[156]; + if ([0x78, 0x67, 0x4c, 0x4b, 0x58].includes(type ?? -1) && size > this.maxMetaEntryBytes) { + throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.META_ENTRY_SIZE_EXCEEDS_LIMIT); + } + if (type === 0x78 || type === 0x67 || type === 0x58) { + throw this.invalid("PAX metadata is unmeterable without interpreting content"); + } + const padded = Math.ceil(size / 512) * 512; + if (!Number.isSafeInteger(padded)) throw this.invalid("entry padding exceeds the safe integer range"); + if (type === 0x53) { + if (this.block[482] !== 0 && this.block[482] !== 1) { + throw this.invalid("GNU sparse extension flag is not 0 or 1"); + } + if (this.block[482] === 0) { + throw this.invalid("GNU sparse entries are not supported"); + } + this.state = { kind: "sparse", dataRemaining: padded, metaBytes: 0 }; + } else { + this.state = padded === 0 ? { kind: "header" } : { kind: "data", remaining: padded }; + } + this.blockLength = 0; + } + + private finishSparseHeader(state: Extract): void { + const metaBytes = state.metaBytes + 512; + if (metaBytes > this.maxMetaEntryBytes) { + throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.META_ENTRY_SIZE_EXCEEDS_LIMIT); + } + if (this.block[504] !== 0 && this.block[504] !== 1) { + throw this.invalid("GNU sparse extension flag is not 0 or 1"); + } + if (this.block[504] === 1) { + this.state = { ...state, metaBytes }; + } else { + throw this.invalid("GNU sparse entries are not supported"); + } + this.blockLength = 0; + } + + private meter(chunk: Buffer): void { + let offset = 0; + while (offset < chunk.length) { + if (this.state.kind === "data") { + const take = Math.min(this.state.remaining, chunk.length - offset); + offset += take; + const remaining = this.state.remaining - take; + this.state = remaining === 0 ? { kind: "header" } : { kind: "data", remaining }; + continue; + } + const take = Math.min(512 - this.blockLength, chunk.length - offset); + chunk.copy(this.block, this.blockLength, offset, offset + take); + this.blockLength += take; + offset += take; + if (this.blockLength === 512) { + if (this.state.kind === "sparse") this.finishSparseHeader(this.state); + else this.finishHeader(); + } + } + } + + override _transform(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null, data?: Buffer) => void): void { + try { + this.meter(chunk); + callback(null, chunk); + } catch (error) { + callback(error instanceof Error ? error : new Error(String(error))); + } + } + + override _flush(callback: (error?: Error | null) => void): void { + if (this.state.kind === "header" && this.blockLength === 0) callback(); + else callback(this.invalid(this.state.kind === "header" ? "truncated TAR header" : "truncated TAR entry")); + } +} + +async function isGzip(filePath: string): Promise { + const handle = await fs.promises.open(filePath, "r"); + try { + const magic = Buffer.alloc(2); + const { bytesRead } = await handle.read(magic, 0, 2, 0); + return bytesRead === 2 && magic[0] === 0x1f && magic[1] === 0x8b; + } finally { + await handle.close(); + } +} + +export async function preflightTarMetadata(params: { + archivePath: string; + maxMetaEntryBytes: number; + signal?: AbortSignal; +}): Promise { + const input = fs.createReadStream(params.archivePath); + const meter = new TarMetadataMeter(params.maxMetaEntryBytes); + const sink = new Writable({ write(_chunk, _encoding, callback) { callback(); } }); + if (await isGzip(params.archivePath)) { + await pipeline(input, createGunzip(), meter, sink, { signal: params.signal }); + } else { + await pipeline(input, meter, sink, { signal: params.signal }); + } +} diff --git a/src/archive-tar-runtime.ts b/src/archive-tar-runtime.ts new file mode 100644 index 0000000..9f995cd --- /dev/null +++ b/src/archive-tar-runtime.ts @@ -0,0 +1,49 @@ +export type TarParserEntry = { + meta?: boolean; + size: number; + type?: string; + resume(): void; +}; + +export type TarParser = NodeJS.WritableStream & { + abort(error: Error): void; + on(event: "ignoredEntry", listener: (entry: TarParserEntry) => void): TarParser; + on(event: "entry", listener: (entry: TarParserEntry) => void): TarParser; + on(event: "meta", listener: (metadata: string) => void): TarParser; + on(event: "error", listener: (error: Error) => void): TarParser; + on(event: "end", listener: () => void): TarParser; +}; + +export type TarModule = { + Parser: new (options: { strict: true; maxMetaEntrySize: number }) => TarParser; + x(options: { + cwd: string; + strip: number; + gzip?: boolean; + signal?: AbortSignal; + preservePaths: false; + noChmod: true; + preserveOwner: false; + strict: true; + maxMetaEntrySize: number; + filter?(this: TarParser, entryPath: string, entry: unknown): boolean; + onReadEntry(this: unknown, entry: unknown): void; + }): TarParser; + t(options: { + file: string; + strict: true; + maxMetaEntrySize: number; + onReadEntry(entry: AsyncIterable & { resume(): void }): void; + }): Promise; +}; + +export async function importOptionalTar(): Promise { + try { + return await import("tar"); + } catch (cause) { + throw new Error( + 'Optional archive dependency "tar" is not installed. Install it to use TAR archive helpers from @openclaw/fs-safe/archive.', + { cause }, + ); + } +} diff --git a/src/archive-tar.ts b/src/archive-tar.ts index efaeb3e..614f3af 100644 --- a/src/archive-tar.ts +++ b/src/archive-tar.ts @@ -5,12 +5,20 @@ import { } from "./archive-entry.js"; import { assertArchiveEntryCountWithinLimit, + assertArchiveEntryPathComponentsWithinLimit, createByteBudgetTracker, resolveExtractLimits, type ArchiveExtractLimits, } from "./archive-limits.js"; +import { + archiveEntryKindFromTarType, + shouldExtractArchiveEntry, + type ArchiveEntryFilter, + type ArchiveFilteredEntryPolicy, +} from "./archive-policy.js"; +import { ArchiveSecurityError } from "./archive-errors.js"; -export type TarEntryInfo = { path: string; type: string; size: number }; +export type TarEntryInfo = { path: string; type: string; size: number; mode?: number }; const BLOCKED_TAR_ENTRY_TYPES = new Set([ "SymbolicLink", @@ -38,7 +46,14 @@ export function readTarEntryInfo(entry: unknown): TarEntryInfo { Number.isFinite((entry as { size: number }).size) ? Math.max(0, Math.floor((entry as { size: number }).size)) : 0; - return { path: p, type: t, size: s }; + const mode = + typeof entry === "object" && + entry !== null && + "mode" in entry && + typeof (entry as { mode?: unknown }).mode === "number" + ? (entry as { mode: number }).mode + : undefined; + return { path: p, type: t, size: s, mode }; } export function createTarEntryPreflightChecker(params: { @@ -46,7 +61,9 @@ export function createTarEntryPreflightChecker(params: { stripComponents?: number; limits?: ArchiveExtractLimits; escapeLabel?: string; -}): (entry: TarEntryInfo) => void { + entryFilter?: ArchiveEntryFilter; + onFiltered?: ArchiveFilteredEntryPolicy; +}): (entry: TarEntryInfo) => boolean { const strip = Math.max(0, Math.floor(params.stripComponents ?? 0)); const limits = resolveExtractLimits(params.limits); let entryCount = 0; @@ -57,9 +74,10 @@ export function createTarEntryPreflightChecker(params: { const relPath = stripArchivePath(entry.path, strip); if (!relPath) { - return; + return false; } validateArchiveEntryPath(relPath, { escapeLabel: params.escapeLabel }); + assertArchiveEntryPathComponentsWithinLimit(relPath, limits); resolveArchiveOutputPath({ rootDir: params.rootDir, relPath, @@ -67,12 +85,25 @@ export function createTarEntryPreflightChecker(params: { escapeLabel: params.escapeLabel, }); + entryCount += 1; + assertArchiveEntryCountWithinLimit(entryCount, limits); + + const kind = archiveEntryKindFromTarType(entry.type); + if ( + !shouldExtractArchiveEntry({ + filter: params.entryFilter, + onFiltered: params.onFiltered, + entry: { path: entry.path, kind, size: entry.size }, + }) + ) { + return false; + } + if (BLOCKED_TAR_ENTRY_TYPES.has(entry.type)) { - throw new Error(`tar entry is a link: ${entry.path}`); + throw new ArchiveSecurityError("entry-link", `tar entry is a link: ${entry.path}`); } - entryCount += 1; - assertArchiveEntryCountWithinLimit(entryCount, limits); budget.addEntrySize(entry.size); + return true; }; } diff --git a/src/archive-zip-entry.ts b/src/archive-zip-entry.ts new file mode 100644 index 0000000..1dd3dd5 --- /dev/null +++ b/src/archive-zip-entry.ts @@ -0,0 +1,38 @@ +import { + resolveArchiveEntryMode, + type ArchiveEntryModePolicy, +} from "./archive-policy.js"; + +export type ZipEntry = { + name: string; + dir: boolean; + unixPermissions?: number; + _data?: { uncompressedSize?: number }; + nodeStream?: () => NodeJS.ReadableStream; + async: (type: "nodebuffer") => Promise; +}; + +const ZIP_UNIX_FILE_TYPE_MASK = 0o170000; +const ZIP_UNIX_SYMLINK_TYPE = 0o120000; + +export function isZipSymlinkEntry(entry: ZipEntry): boolean { + return ( + typeof entry.unixPermissions === "number" && + (entry.unixPermissions & ZIP_UNIX_FILE_TYPE_MASK) === ZIP_UNIX_SYMLINK_TYPE + ); +} + +export function zipEntryMode( + entry: ZipEntry, + policy: ArchiveEntryModePolicy | undefined, +): number { + return resolveArchiveEntryMode({ + kind: entry.dir ? "directory" : "file", + archivedMode: entry.unixPermissions, + policy, + }); +} + +export function zipEntryDeclaredSize(entry: ZipEntry): number { + return Math.max(0, Math.floor(entry._data?.uncompressedSize ?? 0)); +} diff --git a/src/archive.ts b/src/archive.ts index 027cb7d..ca54ed6 100644 --- a/src/archive.ts +++ b/src/archive.ts @@ -1,4 +1,4 @@ -import { constants as fsConstants } from "node:fs"; +import fsSync, { constants as fsConstants } from "node:fs"; import type { FileHandle } from "node:fs/promises"; import fs from "node:fs/promises"; import path from "node:path"; @@ -15,17 +15,15 @@ import { withExtractionDeadline, type ExtractionDeadline, } from "./archive-deadline.js"; -import { writeFileHandleFully } from "./archive-file-io.js"; import { - ARCHIVE_LIMIT_ERROR_CODE, - ArchiveLimitError, assertArchiveEntryCountWithinLimit, + assertArchiveEntryPathComponentsWithinLimit, createByteBudgetTracker, createExtractBudgetTransform, resolveExtractLimits, type ArchiveExtractLimits, } from "./archive-limits.js"; -import { resolveArchiveKind, type ArchiveKind } from "./archive-kind.js"; +import { resolveArchiveKind } from "./archive-kind.js"; import { mergeExtractedTreeIntoDestination, prepareArchiveDestinationDir, @@ -38,15 +36,32 @@ import { type TarEntryInfo, } from "./archive-tar.js"; import { loadZipArchiveWithPreflight } from "./archive-zip-preflight.js"; -import { sameFileIdentity } from "./file-identity.js"; +import { + isZipSymlinkEntry, + zipEntryDeclaredSize, + zipEntryMode, + type ZipEntry, +} from "./archive-zip-entry.js"; +import { FsSafeError } from "./errors.js"; +import { ArchiveSecurityError } from "./archive-errors.js"; +import { extractNativeArchive } from "./archive-native.js"; +import { stageArchiveFileForExtraction } from "./archive-input.js"; +import { getNativeBinding } from "./native.js"; +import { + resolveArchiveEntryMode, + shouldExtractArchiveEntry, +} from "./archive-policy.js"; +import { importOptionalTar } from "./archive-tar-runtime.js"; +import { preflightTarMetadata } from "./archive-tar-meta.js"; +import type { ExtractArchiveOptions } from "./archive-options.js"; import { writeSiblingTempFile } from "./sibling-temp.js"; -import { tempFile } from "./temp-target.js"; - -export type ArchiveLogger = { - info?: (message: string) => void; - warn?: (message: string) => void; -}; - +export type { ArchiveLogger, ExtractArchiveOptions } from "./archive-options.js"; +export type { + ArchiveEntryFilter, + ArchiveEntryKind, + ArchiveEntryModePolicy, + ArchiveFilteredEntryPolicy, +} from "./archive-policy.js"; export { isWindowsDrivePath, normalizeArchiveEntryPath, @@ -55,6 +70,7 @@ export { validateArchiveEntryPath, } from "./archive-entry.js"; export { resolveArchiveKind, resolvePackedRootDir, type ArchiveKind } from "./archive-kind.js"; +export { readArchiveEntry } from "./archive-read.js"; export { ARCHIVE_LIMIT_ERROR_CODE, ArchiveLimitError, @@ -62,9 +78,12 @@ export { DEFAULT_MAX_ENTRIES, DEFAULT_MAX_EXTRACTED_BYTES, DEFAULT_MAX_ENTRY_BYTES, + DEFAULT_MAX_META_ENTRY_BYTES, + DEFAULT_MAX_ENTRY_PATH_COMPONENTS, type ArchiveExtractLimits, type ArchiveLimitErrorCode, } from "./archive-limits.js"; +export { ArchiveFormatError, type ArchiveFormatErrorCode } from "./archive-errors.js"; export { ArchiveSecurityError, type ArchiveSecurityErrorCode } from "./archive-staging.js"; export { createArchiveSymlinkTraversalError, @@ -79,136 +98,13 @@ export { readZipCentralDirectoryEntryCount, type ZipArchiveWithFiles, } from "./archive-zip-preflight.js"; - const SUPPORTS_NOFOLLOW = process.platform !== "win32" && "O_NOFOLLOW" in fsConstants; const OPEN_WRITE_CREATE_FLAGS = fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | (SUPPORTS_NOFOLLOW ? fsConstants.O_NOFOLLOW : 0); - -type ZipEntry = { - name: string; - dir: boolean; - unixPermissions?: number; - nodeStream?: () => NodeJS.ReadableStream; - async: (type: "nodebuffer") => Promise; -}; - type ZipExtractBudget = ReturnType; -type StagedArchiveFile = { path: string; cleanup: () => Promise }; -type TarModule = { - x(options: { - file: string; - cwd: string; - strip: number; - gzip?: boolean; - signal?: AbortSignal; - preservePaths: false; - strict: true; - onReadEntry(this: unknown, entry: unknown): void; - }): Promise; -}; - -const ZIP_UNIX_FILE_TYPE_MASK = 0o170000; -const ZIP_UNIX_SYMLINK_TYPE = 0o120000; - -function isZipSymlinkEntry(entry: ZipEntry): boolean { - return ( - typeof entry.unixPermissions === "number" && - (entry.unixPermissions & ZIP_UNIX_FILE_TYPE_MASK) === ZIP_UNIX_SYMLINK_TYPE - ); -} - -function zipEntryFileMode(entry: ZipEntry): number | undefined { - if (typeof entry.unixPermissions !== "number") { - return undefined; - } - const mode = entry.unixPermissions & 0o777; - return mode === 0 ? undefined : mode; -} - -async function cleanupStagedArchiveFile(staged: StagedArchiveFile | undefined): Promise { - if (staged) { - await staged.cleanup().catch(() => undefined); - } -} - -async function closeFileHandle(handle: FileHandle | undefined): Promise { - if (handle) { - await handle.close().catch(() => undefined); - } -} - -async function stageArchiveFileForExtraction(params: { - archivePath: string; - limits: ReturnType; - deadline: ExtractionDeadline; -}): Promise { - params.deadline.check(); - const sourcePath = path.resolve(params.archivePath); - const initialStat = await fs.lstat(sourcePath); - if (initialStat.isSymbolicLink() || !initialStat.isFile()) { - throw new Error(`archive is not a regular file: ${params.archivePath}`); - } - if (initialStat.size > params.limits.maxArchiveBytes) { - throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.ARCHIVE_SIZE_EXCEEDS_LIMIT); - } - - const noFollow = - process.platform !== "win32" && "O_NOFOLLOW" in fsConstants ? fsConstants.O_NOFOLLOW : 0; - const handle = await fs.open(sourcePath, fsConstants.O_RDONLY | noFollow); - let staged: StagedArchiveFile | undefined; - let output: FileHandle | undefined; - try { - staged = await tempFile({ - prefix: "fs-safe-archive-input", - fileName: path.basename(sourcePath), - }); - const openedStat = await handle.stat(); - const pathStat = await fs.lstat(sourcePath); - if ( - !openedStat.isFile() || - pathStat.isSymbolicLink() || - !pathStat.isFile() || - !sameFileIdentity(initialStat, openedStat) || - !sameFileIdentity(pathStat, openedStat) - ) { - throw new Error("archive changed during validation"); - } - - output = await fs.open(staged.path, OPEN_WRITE_CREATE_FLAGS, 0o600); - const buffer = Buffer.allocUnsafe(64 * 1024); - let written = 0; - while (true) { - params.deadline.check(); - const { bytesRead } = await handle.read(buffer, 0, buffer.length, null); - if (bytesRead === 0) { - break; - } - written += bytesRead; - if (written > params.limits.maxArchiveBytes) { - throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.ARCHIVE_SIZE_EXCEEDS_LIMIT); - } - await writeFileHandleFully({ - handle: output, - buffer, - bytes: bytesRead, - deadline: params.deadline, - }); - params.deadline.check(); - } - await output.close(); - output = undefined; - return staged; - } catch (err) { - await closeFileHandle(output); - await cleanupStagedArchiveFile(staged); - throw err; - } finally { - await closeFileHandle(handle); - } -} async function readZipEntryStream(entry: ZipEntry): Promise { if (typeof entry.nodeStream === "function") { @@ -256,6 +152,7 @@ async function writeZipFileEntry(params: { outPath: string; budget: ZipExtractBudget; deadline: ExtractionDeadline; + mode: number; }): Promise { params.deadline.check(); params.budget.startEntry(); @@ -270,7 +167,7 @@ async function writeZipFileEntry(params: { dir: path.dirname(destinationPath), tempPrefix: `.${path.basename(destinationPath)}.fs-safe-archive`, chmodDir: false, - mode: zipEntryFileMode(params.entry), + mode: params.mode, writeTemp: async (tempPath) => { tempHandle = await fs.open(tempPath, OPEN_WRITE_CREATE_FLAGS, 0o666); const writable = tempHandle.createWriteStream(); @@ -316,6 +213,9 @@ async function extractZip(params: { stripComponents?: number; limits?: ArchiveExtractLimits; deadline: ExtractionDeadline; + entryModes?: ExtractArchiveOptions["entryModes"]; + entryFilter?: ExtractArchiveOptions["entryFilter"]; + onFiltered?: ExtractArchiveOptions["onFiltered"]; }): Promise { const limits = resolveExtractLimits(params.limits); const stagedArchive = await stageArchiveFileForExtraction({ @@ -351,6 +251,21 @@ async function extractZip(params: { if (!output) { continue; } + assertArchiveEntryPathComponentsWithinLimit(output.relPath, limits); + + const isSymlink = isZipSymlinkEntry(entry); + const entryKind = isSymlink ? "symlink" : entry.dir ? "directory" : "file"; + const entrySize = zipEntryDeclaredSize(entry); + if ( + !shouldExtractArchiveEntry({ + filter: params.entryFilter, + onFiltered: params.onFiltered, + entry: { path: entry.name, kind: entryKind, size: entrySize }, + }) + ) { + continue; + } + const mode = zipEntryMode(entry, params.entryModes); await prepareZipOutputPath({ destinationDir: stagingRealDir, @@ -361,10 +276,11 @@ async function extractZip(params: { isDirectory: entry.dir, }); if (entry.dir) { + await fs.chmod(output.outPath, mode); continue; } - if (isZipSymlinkEntry(entry)) { - throw new Error(`zip entry is a link: ${entry.name}`); + if (isSymlink) { + throw new ArchiveSecurityError("entry-link", `zip entry is a link: ${entry.name}`); } await writeZipFileEntry({ @@ -372,6 +288,7 @@ async function extractZip(params: { outPath: output.outPath, budget, deadline: params.deadline, + mode, }); } @@ -389,22 +306,37 @@ async function extractZip(params: { } } -export async function extractArchive(params: { - archivePath: string; - destDir: string; - timeoutMs: number; - kind?: ArchiveKind; - stripComponents?: number; - tarGzip?: boolean; - limits?: ArchiveExtractLimits; - logger?: ArchiveLogger; -}): Promise { +export async function extractArchive(params: ExtractArchiveOptions): Promise { const kind = params.kind ?? resolveArchiveKind(params.archivePath); if (!kind) { throw new Error(`unsupported archive: ${params.archivePath}`); } const label = kind === "zip" ? "extract zip" : "extract tar"; + const native = getNativeBinding(); + if (native) { + await withExtractionDeadline(params.timeoutMs, label, async (deadline) => + extractNativeArchive({ + binding: native, + archivePath: params.archivePath, + destDir: params.destDir, + kind, + stripComponents: params.stripComponents, + limits: params.limits, + deadline, + entryModes: params.entryModes, + entryFilter: params.entryFilter, + onFiltered: params.onFiltered, + }), + ); + return; + } + if (kind === "tar-zstd" || kind === "tar-bzip2") { + throw new FsSafeError( + "helper-unavailable", + `${kind} archives require the optional native binding; install the matching platform package`, + ); + } if (kind === "tar") { await withExtractionDeadline(params.timeoutMs, label, async (deadline) => { const tar = await importOptionalTar(); @@ -415,6 +347,12 @@ export async function extractArchive(params: { deadline, }); try { + await preflightTarMetadata({ + archivePath: stagedArchive.path, + maxMetaEntryBytes: limits.maxMetaEntryBytes, + signal: deadline.signal, + }); + deadline.check(); const destinationRealDir = await prepareArchiveDestinationDir(params.destDir); await withStagedArchiveDestination({ destinationRealDir, @@ -424,23 +362,59 @@ export async function extractArchive(params: { rootDir: destinationRealDir, stripComponents: params.stripComponents, limits, + entryFilter: params.entryFilter, + onFiltered: params.onFiltered, }); + const acceptedEntries: Array<{ path: string; mode: number }> = []; // A canonical cwd is not enough here: tar can still follow // pre-existing child symlinks in the live destination tree. // Extract into a private staging dir first, then merge through // the same safe-open boundary checks used by direct file writes. - await tar.x({ - file: stagedArchive.path, + const extractor = tar.x({ cwd: stagingDir, strip: Math.max(0, Math.floor(params.stripComponents ?? 0)), gzip: params.tarGzip, signal: deadline.signal, preservePaths: false, + noChmod: true, + preserveOwner: false, strict: true, + maxMetaEntrySize: limits.maxMetaEntryBytes, + filter(this: { abort(error: Error): void }, _entryPath, entry) { + try { + const info = readTarEntryInfo(entry); + const accepted = checkTarEntrySafety(info); + if (accepted) { + const relPath = stripArchivePath( + info.path, + Math.max(0, Math.floor(params.stripComponents ?? 0)), + ); + if (relPath) { + acceptedEntries.push({ + path: relPath, + mode: resolveArchiveEntryMode({ + kind: + info.type === "Directory" || info.type === "GNUDumpDir" + ? "directory" + : "file", + archivedMode: info.mode, + policy: params.entryModes, + }), + }); + } + } + return accepted; + } catch (error) { + // Abort through the parser so pipeline tears down both the + // archive reader and unpacker instead of leaving a paused + // stream behind after a policy rejection. + this.abort(error instanceof Error ? error : new Error(String(error))); + return false; + } + }, onReadEntry(entry) { try { deadline.check(); - checkTarEntrySafety(readTarEntryInfo(entry)); } catch (err) { const error = err instanceof Error ? err : new Error(String(err)); // Node's EventEmitter calls listeners with `this` bound to the @@ -450,6 +424,21 @@ export async function extractArchive(params: { } }, }); + try { + await pipeline(fsSync.createReadStream(stagedArchive.path), extractor, { + signal: deadline.signal, + }); + } catch (error) { + throw createPipelineTimeoutError(error, deadline); + } + for (const accepted of acceptedEntries) { + const outputPath = resolveArchiveOutputPath({ + rootDir: stagingDir, + relPath: accepted.path, + originalPath: accepted.path, + }); + await fs.chmod(outputPath, accepted.mode); + } deadline.check(); await mergeExtractedTreeIntoDestination({ sourceDir: stagingDir, @@ -473,21 +462,9 @@ export async function extractArchive(params: { stripComponents: params.stripComponents, limits: params.limits, deadline, + entryModes: params.entryModes, + entryFilter: params.entryFilter, + onFiltered: params.onFiltered, }), ); } - -async function importOptionalTar(): Promise { - try { - return await import("tar"); - } catch (err) { - throw missingOptionalArchiveDependencyError("tar", err); - } -} - -function missingOptionalArchiveDependencyError(packageName: "tar", cause: unknown): Error { - return new Error( - `Optional archive dependency "${packageName}" is not installed. Install it to use TAR archive helpers from @openclaw/fs-safe/archive.`, - { cause }, - ); -} diff --git a/src/config.ts b/src/config.ts index 52550e9..a0ee0ba 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,9 +1,11 @@ export { configureFsSafePython, - getFsSafePythonConfig, + configureFsSafeNative, + getFsSafeNativeConfig, type FsSafePythonConfig, - type FsSafePythonMode, -} from "./pinned-python-config.js"; + type FsSafeNativeConfig, + type FsSafeNativeMode, +} from "./native-config.js"; export { configureFsSafeLocks, getFsSafeLockConfig, diff --git a/src/durability.ts b/src/durability.ts index 16dd054..cea493e 100644 --- a/src/durability.ts +++ b/src/durability.ts @@ -11,3 +11,19 @@ export { type EnsureDurableDirectoryOptions, type PinnedDirectory, } from "./directory-durability.js"; +export { + isHardlinkFallbackError, + publishFileExclusive, + type PublishFileExclusiveResult, + type PublishFileExclusiveStrategy, + type PublishFileExclusiveCleanup, + type PublishFileExclusiveDirectorySyncFailure, + type PublishFileExclusiveFailureDetails, + type PublishFileExclusiveFailurePhase, + type PublishFileExclusiveSyncFailurePolicy, +} from "./publish-file.js"; +export { + sha256File, + type Sha256FileInput, + type Sha256FileResult, +} from "./file-hash.js"; diff --git a/src/errors.ts b/src/errors.ts index fde0db5..882ab88 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -16,12 +16,14 @@ export type FsSafeErrorCode = | "path-alias" | "path-mismatch" | "permission-unverified" + | "secret-exists" | "symlink" | "timeout" | "too-large" | "unsupported-platform"; export type FsSafeErrorCategory = "policy" | "operational"; +export type FsSafeErrorDetails = Readonly>; const OPERATIONAL_CODES: ReadonlySet = new Set([ "helper-failed", @@ -38,11 +40,17 @@ export function categorizeFsSafeError(code: FsSafeErrorCode): FsSafeErrorCategor export class FsSafeError extends Error { readonly code: FsSafeErrorCode; readonly category: FsSafeErrorCategory; + readonly details?: FsSafeErrorDetails; - constructor(code: FsSafeErrorCode, message: string, options: { cause?: unknown } = {}) { + constructor( + code: FsSafeErrorCode, + message: string, + options: { cause?: unknown; details?: FsSafeErrorDetails } = {}, + ) { super(message, options); this.name = "FsSafeError"; this.code = code; this.category = categorizeFsSafeError(code); + this.details = options.details; } } diff --git a/src/file-hash.ts b/src/file-hash.ts new file mode 100644 index 0000000..b364f49 --- /dev/null +++ b/src/file-hash.ts @@ -0,0 +1,91 @@ +import { createHash } from "node:crypto"; +import fsSync from "node:fs"; +import type { FileHandle } from "node:fs/promises"; +import fs from "node:fs/promises"; +import { FsSafeError } from "./errors.js"; +import { sameFileIdentity } from "./file-identity.js"; +import { getNativeBinding, type NativeBinding } from "./native.js"; + +export type Sha256FileInput = string | FileHandle; + +export type Sha256FileResult = { + bytes: number; + digest: string; +}; + +function readFlags(): number { + return ( + fsSync.constants.O_RDONLY | + (process.platform !== "win32" && typeof fsSync.constants.O_NOFOLLOW === "number" + ? fsSync.constants.O_NOFOLLOW + : 0) | + (process.platform !== "win32" && typeof fsSync.constants.O_NONBLOCK === "number" + ? fsSync.constants.O_NONBLOCK + : 0) + ); +} + +export async function hashFileHandle( + handle: FileHandle, + native: NativeBinding | undefined = getNativeBinding(), +): Promise { + const stat = await handle.stat(); + if (!stat.isFile()) { + throw new FsSafeError("not-file", "SHA-256 input is not a regular file"); + } + if (native) { + return await native.sha256File(handle.fd); + } + + const hash = createHash("sha256"); + const buffer = Buffer.allocUnsafe(64 * 1024); + let position = 0; + while (true) { + const { bytesRead } = await handle.read(buffer, 0, buffer.length, position); + if (bytesRead === 0) { + return { bytes: position, digest: hash.digest("hex") }; + } + hash.update(buffer.subarray(0, bytesRead)); + position += bytesRead; + } +} + +async function hashPath(filePath: string): Promise { + const before = await fs.lstat(filePath); + if (before.isSymbolicLink()) { + throw new FsSafeError("symlink", "SHA-256 path must not be a symbolic link"); + } + if (!before.isFile()) { + throw new FsSafeError("not-file", "SHA-256 path is not a regular file"); + } + + let handle: FileHandle; + try { + handle = await fs.open(filePath, readFlags()); + } catch (error) { + if ((error as NodeJS.ErrnoException | null)?.code === "ELOOP") { + throw new FsSafeError("symlink", "SHA-256 path must not be a symbolic link", { + cause: error, + }); + } + throw error; + } + + try { + const opened = await handle.stat(); + const current = await fs.lstat(filePath); + if (!opened.isFile()) { + throw new FsSafeError("not-file", "SHA-256 path is not a regular file"); + } + if (current.isSymbolicLink() || !current.isFile() || !sameFileIdentity(opened, current)) { + throw new FsSafeError("path-mismatch", "SHA-256 path changed while opening"); + } + return await hashFileHandle(handle); + } finally { + await handle.close().catch(() => undefined); + } +} + +export async function sha256File(input: Sha256FileInput): Promise { + return typeof input === "string" ? await hashPath(input) : await hashFileHandle(input); +} diff --git a/src/file-lock-sync.ts b/src/file-lock-sync.ts new file mode 100644 index 0000000..eafe6ac --- /dev/null +++ b/src/file-lock-sync.ts @@ -0,0 +1,231 @@ +import fs from "node:fs"; +import path from "node:path"; +import { FsSafeError } from "./errors.js"; +import type { Root } from "./root-impl.js"; +import { + readSidecarLockSnapshotSync, + relativeSidecarLockPath, + removeSidecarLockIfUnchangedSync, + serializeSidecarLockPayload, + sidecarLockSnapshotMatches, + type SidecarLockSnapshot, + type SidecarLockStaleSnapshot, +} from "./sidecar-lock-reclaim.js"; +import { + computeSidecarLockDelayMs, + sidecarLockPayloadIsStale, +} from "./sidecar-lock-policy.js"; +import type { + SidecarLockCompromisedInfo, + SidecarLockRetryOptions, + SidecarLockStaleRecovery, +} from "./sidecar-lock-types.js"; + +export type FileLockSyncAcquireOptions> = { + lockPath?: string; + staleMs?: number; + timeoutMs?: number; + retry?: SidecarLockRetryOptions; + staleRecovery?: SidecarLockStaleRecovery; + payload: () => TPayload; + shouldReclaim?: (params: { + lockPath: string; + normalizedTargetPath: string; + payload: unknown; + staleMs: number; + nowMs: number; + heldByThisProcess: false; + }) => boolean; + shouldRemoveStaleLock?: (snapshot: SidecarLockStaleSnapshot) => boolean; + parsePayload?: (raw: string) => unknown; + lockRoot?: Root; + onCompromised?: (info: SidecarLockCompromisedInfo) => void; + compromiseCheckIntervalMs?: number; +}; + +export type FileLockSyncHandle = { + lockPath: string; + normalizedTargetPath: string; + verifyStillHeld(): boolean; + release(): void; + [Symbol.dispose](): void; +}; + +function normalizeTargetPath(targetPath: string): string { + const resolved = path.resolve(targetPath); + fs.mkdirSync(path.dirname(resolved), { recursive: true }); + try { + return path.join(fs.realpathSync(path.dirname(resolved)), path.basename(resolved)); + } catch { + return resolved; + } +} + +function boundedLockPath(lockPath: string, lockRoot?: Root): string { + const resolved = path.resolve(lockPath); + if (!lockRoot) return resolved; + relativeSidecarLockPath(lockRoot, resolved); + const parent = path.dirname(resolved); + const parentReal = fs.realpathSync(parent); + const parentRelative = path.relative(lockRoot.rootReal, parentReal); + if (parentRelative === ".." || parentRelative.startsWith(`..${path.sep}`) || path.isAbsolute(parentRelative)) { + throw new FsSafeError("outside-workspace", "sidecar lock parent is outside lockRoot"); + } + return path.join(parentReal, path.basename(resolved)); +} + +function defaultShouldReclaim(payload: unknown, lockPath: string, staleMs: number, nowMs: number): boolean { + if (sidecarLockPayloadIsStale(payload, staleMs, nowMs)) return true; + try { + return nowMs - fs.statSync(lockPath).mtimeMs > staleMs; + } catch { + return true; + } +} + +function sleep(milliseconds: number): void { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds); +} + +export function acquireFileLockSync>( + targetPath: string, + options: FileLockSyncAcquireOptions, +): FileLockSyncHandle { + const normalizedTargetPath = normalizeTargetPath(targetPath); + const lockPath = boundedLockPath(options.lockPath ?? `${normalizedTargetPath}.lock`, options.lockRoot); + const staleMs = options.staleMs ?? 30_000; + const retry = options.retry ?? {}; + const startedAt = Date.now(); + let attempt = 0; + + while (true) { + let fd: number | undefined; + try { + const payload = options.payload(); + const { raw, ownershipToken } = serializeSidecarLockPayload(payload); + const noFollow = + process.platform !== "win32" && typeof fs.constants.O_NOFOLLOW === "number" + ? fs.constants.O_NOFOLLOW + : 0; + fd = fs.openSync( + lockPath, + fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | noFollow, + 0o600, + ); + fs.writeFileSync(fd, raw, "utf8"); + fs.fsyncSync(fd); + const snapshot: SidecarLockSnapshot = { + raw, + payload, + stat: fs.fstatSync(fd), + ownershipToken, + }; + const heldFd = fd; + let released = false; + let timer: NodeJS.Timeout | undefined; + const verifyStillHeld = () => { + const current = readSidecarLockSnapshotSync(lockPath, options.parsePayload); + return !!current && sidecarLockSnapshotMatches(current, snapshot); + }; + const release = () => { + if (released) return; + released = true; + if (timer) clearInterval(timer); + fs.closeSync(heldFd); + fd = undefined; + removeSidecarLockIfUnchangedSync(lockPath, snapshot); + }; + if (options.onCompromised && (options.compromiseCheckIntervalMs ?? 0) > 0) { + timer = setInterval(() => { + if (!verifyStillHeld()) { + if (timer) clearInterval(timer); + timer = undefined; + options.onCompromised?.({ lockPath, normalizedTargetPath }); + } + }, options.compromiseCheckIntervalMs); + timer.unref(); + } + return { + lockPath, + normalizedTargetPath, + verifyStillHeld, + release, + [Symbol.dispose]: release, + }; + } catch (error) { + if (fd !== undefined) { + const failed = { payload: null, stat: fs.fstatSync(fd) } satisfies SidecarLockSnapshot; + fs.closeSync(fd); + fd = undefined; + removeSidecarLockIfUnchangedSync(lockPath, failed); + } + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + const snapshot = readSidecarLockSnapshotSync(lockPath, options.parsePayload); + if (!snapshot) continue; + const nowMs = Date.now(); + const reclaim = options.shouldReclaim + ? options.shouldReclaim({ + lockPath, + normalizedTargetPath, + payload: snapshot.payload, + staleMs, + nowMs, + heldByThisProcess: false, + }) + : defaultShouldReclaim(snapshot.payload, lockPath, staleMs, nowMs); + if (reclaim) { + if ( + options.staleRecovery === "remove-if-unchanged" && + snapshot.raw !== undefined && + options.shouldRemoveStaleLock?.({ + lockPath, + normalizedTargetPath, + raw: snapshot.raw, + payload: snapshot.payload, + }) + ) { + const reclaimGuard = `${lockPath}.reclaim`; + try { + fs.mkdirSync(reclaimGuard); + if (removeSidecarLockIfUnchangedSync(lockPath, snapshot)) continue; + } finally { + try { + fs.rmdirSync(reclaimGuard); + } catch { + // A surviving reclaim guard fails closed. + } + } + } + throw Object.assign(new Error(`file lock stale for ${normalizedTargetPath}`), { + code: "file_lock_stale", + lockPath, + normalizedTargetPath, + }); + } + const elapsed = Date.now() - startedAt; + const timedOut = options.timeoutMs !== undefined && elapsed >= options.timeoutMs; + if (timedOut || (retry.retries !== undefined && attempt >= retry.retries)) { + throw Object.assign(new Error(`file lock timeout for ${normalizedTargetPath}`), { + code: "file_lock_timeout", + lockPath, + normalizedTargetPath, + }); + } + sleep(computeSidecarLockDelayMs(retry, attempt)); + attempt += 1; + } + } +} + +export function withFileLockSync>( + targetPath: string, + options: FileLockSyncAcquireOptions, + fn: () => T, +): T { + const lock = acquireFileLockSync(targetPath, options); + try { + return fn(); + } finally { + lock.release(); + } +} diff --git a/src/file-lock.ts b/src/file-lock.ts index 972c78f..a2d845e 100644 --- a/src/file-lock.ts +++ b/src/file-lock.ts @@ -7,6 +7,12 @@ import { type SidecarLockStaleRecovery, } from "./sidecar-lock.js"; import { getFsSafeLockConfig } from "./lock-config.js"; +export { + acquireFileLockSync, + withFileLockSync, + type FileLockSyncAcquireOptions, + type FileLockSyncHandle, +} from "./file-lock-sync.js"; export type FileLockRetryOptions = SidecarLockRetryOptions; export type FileLockStaleRecovery = SidecarLockStaleRecovery; @@ -21,6 +27,7 @@ export type FileLockAcquireOptions> = O export type FileLockHandle = SidecarLockHandle; export type FileLockHeldEntry = SidecarLockHeldEntry; +export type { SidecarLockCompromisedInfo } from "./sidecar-lock.js"; export type FileLockManager = { acquire>( diff --git a/src/guarded-mkdir.ts b/src/guarded-mkdir.ts index 15f4a3e..2adfa0b 100644 --- a/src/guarded-mkdir.ts +++ b/src/guarded-mkdir.ts @@ -80,6 +80,7 @@ export async function mkdirPathComponentsWithGuards(params: { throw new FsSafeError("not-file", "directory component must be a directory"); } await createAsyncDirectoryGuard(nextReal); + await assertAsyncDirectoryGuard(parentGuard); current = nextReal; continue; } diff --git a/src/index.ts b/src/index.ts index b139463..ed7b5ce 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,7 @@ export { categorizeFsSafeError, type FsSafeErrorCategory, type FsSafeErrorCode, + type FsSafeErrorDetails, } from "./errors.js"; export { DEFAULT_ROOT_MAX_BYTES, @@ -27,16 +28,28 @@ export { type RootRemoveOptions, type RootWriteJsonOptions, type RootWriteOptions, + type RootWalkDataEntry, + type RootWalkDataEntryKind, + type RootWalkDirectoryErrorBehavior, + type RootWalkEntry, + type RootWalkEntryFilter, + type RootWalkEntryFilterResult, + type RootWalkEntryKind, + type RootWalkLimitBehavior, + type RootWalkOptions, + type RootWalkSymlinkPolicy, type SymlinkPolicy, type WritableOpenMode, type WritableOpenResult, } from "./root.js"; export { configureFsSafePython, - getFsSafePythonConfig, + configureFsSafeNative, + getFsSafeNativeConfig, type FsSafePythonConfig, - type FsSafePythonMode, -} from "./pinned-python-config.js"; + type FsSafeNativeConfig, + type FsSafeNativeMode, +} from "./native-config.js"; export { writeExternalFileWithinRoot, type ExternalFileWriteOptions, diff --git a/src/move-path.ts b/src/move-path.ts index b10936b..38d5e0c 100644 --- a/src/move-path.ts +++ b/src/move-path.ts @@ -3,6 +3,7 @@ import { constants as fsConstants } from "node:fs"; import type { FileHandle } from "node:fs/promises"; import fs from "node:fs/promises"; import path from "node:path"; +import { FsSafeError } from "./errors.js"; import { guardedRename } from "./guarded-mutation.js"; export type MovePathWithCopyFallbackOptions = { @@ -33,6 +34,7 @@ type EntryIdentity = { ino: number; mode: number; mtimeMs: number; + nlink: number; size: number; }; @@ -45,12 +47,68 @@ type CopiedEntryManifest = type CleanupCopiedEntryResult = "removed" | "stale"; +const MAX_HARDLINK_PREFLIGHT_ENTRIES = 50_000; + +function hardlinkedSourceError(sourcePath: string): FsSafeError { + return new FsSafeError("hardlink", `Refusing to move hardlinked file: ${sourcePath}`); +} + +function hardlinkWalkTooLargeError(): FsSafeError { + return new FsSafeError( + "too-large", + `Source hardlink preflight exceeds ${MAX_HARDLINK_PREFLIGHT_ENTRIES} entries`, + ); +} + +async function preflightSourceHardlinks(sourcePath: string): Promise { + const pending = [sourcePath]; + let discovered = 1; + + while (pending.length > 0) { + const current = pending.pop()!; + const stat = await fs.lstat(current); + if (stat.isFile() && stat.nlink > 1) { + throw hardlinkedSourceError(current); + } + if (!stat.isDirectory()) { + continue; + } + const directory = await fs.opendir(current); + for await (const entry of directory) { + discovered += 1; + if (discovered > MAX_HARDLINK_PREFLIGHT_ENTRIES) { + throw hardlinkWalkTooLargeError(); + } + pending.push(path.join(current, entry.name)); + } + } +} + +function isSameOrDescendant(parentPath: string, candidatePath: string): boolean { + const relative = path.relative(parentPath, candidatePath); + return ( + relative === "" || + (!path.isAbsolute(relative) && !relative.startsWith(`..${path.sep}`) && relative !== "..") + ); +} + +async function assertCopyDestinationOutsideSource(sourcePath: string, targetPath: string) { + const sourceReal = await fs.realpath(sourcePath); + const normalizedTarget = path.resolve(targetPath); + const targetParentReal = await fs.realpath(path.dirname(normalizedTarget)); + const targetCandidate = path.join(targetParentReal, path.basename(normalizedTarget)); + if (isSameOrDescendant(sourceReal, targetCandidate)) { + throw new FsSafeError("invalid-path", "Move destination must not be inside the source"); + } +} + function entryIdentity(stat: { ctimeMs: number; dev: number; ino: number; mode: number; mtimeMs: number; + nlink: number; size: number; }): EntryIdentity { return { @@ -59,6 +117,7 @@ function entryIdentity(stat: { ino: stat.ino, mode: stat.mode, mtimeMs: stat.mtimeMs, + nlink: stat.nlink, size: stat.size, }; } @@ -68,6 +127,7 @@ function sameIdentity(a: EntryIdentity, b: EntryIdentity): boolean { a.dev === b.dev && a.ino === b.ino && a.mode === b.mode && + a.nlink === b.nlink && a.size === b.size && a.mtimeMs === b.mtimeMs && a.ctimeMs === b.ctimeMs @@ -102,6 +162,9 @@ function regularReadFlags(): number { fsConstants.O_RDONLY | (typeof fsConstants.O_NOFOLLOW === "number" && process.platform !== "win32" ? fsConstants.O_NOFOLLOW + : 0) | + (typeof fsConstants.O_NONBLOCK === "number" && process.platform !== "win32" + ? fsConstants.O_NONBLOCK : 0) ); } @@ -118,6 +181,7 @@ async function copyRegularFilePinned(params: { from: string; identity: EntryIdentity; mode: number; + rejectHardlinks: boolean; to: string; }): Promise { let destinationCreated = false; @@ -133,6 +197,9 @@ async function copyRegularFilePinned(params: { } try { const openedStat = await sourceHandle.stat(); + if (params.rejectHardlinks && openedStat.nlink > 1) { + throw hardlinkedSourceError(params.from); + } if (!openedStat.isFile() || !sameIdentity(params.identity, entryIdentity(openedStat))) { throw sourceChangedError(params.from); } @@ -158,7 +225,11 @@ async function copyRegularFilePinned(params: { // Re-check the opened handle before the staged tree can be committed. If // the source changed while we copied, the caller should retry the move. - if (!sameIdentity(params.identity, entryIdentity(await sourceHandle.stat()))) { + const finalSourceStat = await sourceHandle.stat(); + if (params.rejectHardlinks && finalSourceStat.nlink > 1) { + throw hardlinkedSourceError(params.from); + } + if (!sameIdentity(params.identity, entryIdentity(finalSourceStat))) { throw sourceChangedError(params.from); } await fs.chmod(params.to, modeBits(params.mode)).catch(() => undefined); @@ -175,7 +246,10 @@ async function copyRegularFilePinned(params: { async function copyEntryWithManifest( from: string, to: string, - options: { sourceHardlinks: "allow" | "reject" }, + options: { + sourceHardlinks: "allow" | "reject"; + budget?: { discovered: number }; + }, ): Promise { const sourceStat = await fs.lstat(from); const identity = entryIdentity(sourceStat); @@ -191,7 +265,15 @@ async function copyEntryWithManifest( if (sourceStat.isDirectory()) { await fs.mkdir(to, { mode: modeBits(sourceStat.mode) || 0o755 }); const children: Array<{ name: string; manifest: CopiedEntryManifest }> = []; - for (const child of await fs.readdir(from)) { + const childNames: string[] = []; + const directory = await fs.opendir(from); + for await (const entry of directory) { + if (options.budget && ++options.budget.discovered > MAX_HARDLINK_PREFLIGHT_ENTRIES) { + throw hardlinkWalkTooLargeError(); + } + childNames.push(entry.name); + } + for (const child of childNames) { children.push({ name: child, manifest: await copyEntryWithManifest(path.join(from, child), path.join(to, child), options), @@ -210,10 +292,16 @@ async function copyEntryWithManifest( throw new Error(`Refusing to move non-file path with copy fallback: ${from}`); } if (options.sourceHardlinks === "reject" && sourceStat.nlink > 1) { - throw new Error(`Refusing to move hardlinked file with copy fallback: ${from}`); + throw hardlinkedSourceError(from); } - await copyRegularFilePinned({ from, identity, mode: sourceStat.mode, to }); + await copyRegularFilePinned({ + from, + identity, + mode: sourceStat.mode, + rejectHardlinks: options.sourceHardlinks === "reject", + to, + }); return { ...identity, kind: "leaf" }; } @@ -273,26 +361,40 @@ async function cleanupCopiedEntry( export async function movePathWithCopyFallback( options: MovePathWithCopyFallbackOptions, ): Promise { - let fallbackReason: MoveCopyFallbackReason | undefined; - try { - await guardedRename({ from: options.from, to: options.to }); - return; - } catch (error) { - fallbackReason = moveCopyFallbackReasonForRenameError(error); - if (!fallbackReason) { - throw error; + const sourcePath = path.resolve(options.from); + const targetPath = path.resolve(options.to); + const rejectHardlinks = options.sourceHardlinks === "reject"; + if (rejectHardlinks) { + await preflightSourceHardlinks(sourcePath); + } + + if (!rejectHardlinks) { + try { + await guardedRename({ from: sourcePath, to: targetPath }); + return; + } catch (error) { + if (!moveCopyFallbackReasonForRenameError(error)) { + throw error; + } } + } else { + // A pathname preflight cannot make nlink and rename one atomic operation. + // Commit a fresh inode through the copy path so a post-scan hardlink can + // never become the published target; the copy loop fences nlink again. } - const targetDir = path.dirname(path.resolve(options.to)); + await assertCopyDestinationOutsideSource(sourcePath, targetPath); + const targetDir = path.dirname(targetPath); const staged = path.join(targetDir, `.fs-safe-move-${process.pid}-${randomUUID()}.tmp`); try { - const manifest = await copyEntryWithManifest(options.from, staged, { + const manifest = await copyEntryWithManifest(sourcePath, staged, { sourceHardlinks: options.sourceHardlinks ?? "allow", + ...(rejectHardlinks ? { budget: { discovered: 1 } } : {}), }); - await guardedRename({ from: staged, to: options.to }); - const cleanupResult = await cleanupCopiedEntry(options.from, manifest); + await assertCopyDestinationOutsideSource(sourcePath, targetPath); + await guardedRename({ from: staged, to: targetPath }); + const cleanupResult = await cleanupCopiedEntry(sourcePath, manifest); if (cleanupResult === "stale") { - throw sourceChangedError(options.from); + throw sourceChangedError(sourcePath); } } finally { await fs.rm(staged, { recursive: true, force: true }).catch(() => undefined); diff --git a/src/native-config.ts b/src/native-config.ts new file mode 100644 index 0000000..9587916 --- /dev/null +++ b/src/native-config.ts @@ -0,0 +1,97 @@ +export type FsSafeNativeMode = "auto" | "off" | "require"; + +export type FsSafeNativeConfig = { + mode: FsSafeNativeMode; +}; + +/** @deprecated Compatibility bridge for 0.4 upgrades. Use {@link FsSafeNativeConfig}. */ +export type FsSafePythonConfig = { + mode: FsSafeNativeMode; + pythonPath?: string; +}; + +let overrideConfig: Partial = {}; +let legacyWarningEmitted = false; + +const legacyModeEnvKeys = ["FS_SAFE_PYTHON_MODE", "OPENCLAW_FS_SAFE_PYTHON_MODE"] as const; +const legacyPathEnvKeys = [ + "FS_SAFE_PYTHON", + "OPENCLAW_FS_SAFE_PYTHON", + "OPENCLAW_PINNED_PYTHON", + "OPENCLAW_PINNED_WRITE_PYTHON", +] as const; + +function parseMode(value: string | undefined): FsSafeNativeMode | undefined { + if (!value) { + return undefined; + } + const normalized = value.trim().toLowerCase(); + if (normalized === "0" || normalized === "false" || normalized === "off" || normalized === "never") { + return "off"; + } + if (normalized === "1" || normalized === "true" || normalized === "on" || normalized === "auto") { + return "auto"; + } + if (normalized === "required" || normalized === "require") { + return "require"; + } + return undefined; +} + +export function configureFsSafeNative(config: Partial): void { + overrideConfig = { ...overrideConfig, ...config }; +} + +function warnLegacyPythonConfiguration(source: string, mappedMode: FsSafeNativeMode): void { + if (legacyWarningEmitted) { + return; + } + legacyWarningEmitted = true; + process.emitWarning( + `${source} is a deprecated 0.4 compatibility bridge; mapped to native mode ` + + `"${mappedMode}". Use configureFsSafeNative({ mode: "${mappedMode}" }) or ` + + `FS_SAFE_NATIVE_MODE=${mappedMode}. Python interpreter paths are no longer used.`, + { code: "FS_SAFE_PYTHON_DEPRECATED", type: "DeprecationWarning" }, + ); +} + +function readLegacyPythonMode(): FsSafeNativeMode | undefined { + const configuredKeys = [...legacyModeEnvKeys, ...legacyPathEnvKeys].filter( + (key) => process.env[key] !== undefined, + ); + if (configuredKeys.length === 0) { + return undefined; + } + const rawMode = legacyModeEnvKeys.map((key) => process.env[key]).find((value) => value !== undefined); + const mappedMode = parseMode(rawMode) ?? "auto"; + warnLegacyPythonConfiguration(`Legacy ${configuredKeys.join(", ")}`, mappedMode); + return mappedMode; +} + +/** + * @deprecated Compatibility bridge for 0.4 upgrades. Use configureFsSafeNative. + */ +export function configureFsSafePython(config: Partial): void { + const mappedMode = config.mode ?? "auto"; + warnLegacyPythonConfiguration("configureFsSafePython()", mappedMode); + if (config.mode !== undefined) { + configureFsSafeNative({ mode: config.mode }); + } +} + +export function getFsSafeNativeConfig(): FsSafeNativeConfig { + const legacyMode = readLegacyPythonMode(); + return { + mode: + overrideConfig.mode ?? + parseMode(process.env.FS_SAFE_NATIVE_MODE) ?? + parseMode(process.env.OPENCLAW_FS_SAFE_NATIVE_MODE) ?? + legacyMode ?? + "auto", + }; +} + +export function __resetFsSafeNativeConfigForTest(): void { + overrideConfig = {}; + legacyWarningEmitted = false; +} diff --git a/src/native-operations.ts b/src/native-operations.ts new file mode 100644 index 0000000..18398a1 --- /dev/null +++ b/src/native-operations.ts @@ -0,0 +1,144 @@ +import fsSync, { type Stats } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { sameFileIdentity } from "./file-identity.js"; +import { getNativeBinding, type NativeBinding } from "./native.js"; + +export type NativeFileHandle = { + readonly fd: number; + close(): Promise; + stat(): Promise; + writeFile(data: string | Buffer, encoding?: BufferEncoding): Promise; +}; + +function nativeOpenFlags(flags: number): number { + const closeOnExec = (fsSync.constants as typeof fsSync.constants & { O_CLOEXEC?: number }).O_CLOEXEC; + return ( + flags | + (closeOnExec ?? 0) | + (typeof fsSync.constants.O_NOFOLLOW === "number" ? fsSync.constants.O_NOFOLLOW : 0) + ); +} + +function writeAll(fd: number, data: Buffer): void { + let offset = 0; + while (offset < data.byteLength) { + const written = fsSync.writeSync(fd, data, offset, data.byteLength - offset); + if (written <= 0) { + throw Object.assign(new Error("native file write made no progress"), { code: "EIO" }); + } + offset += written; + } +} + +function wrapNativeFd(fd: number): NativeFileHandle { + let open = true; + return { + fd, + async close() { + if (open) { + open = false; + fsSync.closeSync(fd); + } + }, + async stat() { + return fsSync.fstatSync(fd); + }, + async writeFile(data, encoding) { + writeAll(fd, Buffer.isBuffer(data) ? data : Buffer.from(data, encoding ?? "utf8")); + }, + }; +} + +export function removeNativeCreatedFileIfStillPinned(params: { + binding: NativeBinding; + parentPath: string; + parentFd: number; + basename: string; + created?: Stats; +}): void { + if (!params.created) { + return; + } + try { + const parentPathStat = fsSync.lstatSync(params.parentPath); + const parentFdStat = params.binding.fstatIdentity(params.parentFd); + const targetPath = path.join(params.parentPath, params.basename); + const target = fsSync.lstatSync(targetPath); + if ( + !parentPathStat.isSymbolicLink() && + parentPathStat.dev === parentFdStat.dev && + parentPathStat.ino === parentFdStat.ino && + !target.isSymbolicLink() && + sameFileIdentity(target, params.created) + ) { + fsSync.rmSync(targetPath); + } + } catch { + // Failed cleanup leaves a fail-closed artifact instead of deleting an + // unverified pathname. + } +} + +export async function createNativeExclusiveFile( + targetPath: string, + mode: number, +): Promise { + const binding = getNativeBinding(); + if (!binding) { + return undefined; + } + const parentPath = path.dirname(targetPath); + const basename = path.basename(targetPath); + const parent = await fs.open( + parentPath, + fsSync.constants.O_RDONLY | + (typeof fsSync.constants.O_DIRECTORY === "number" ? fsSync.constants.O_DIRECTORY : 0), + ); + let fd: number | undefined; + let created: Stats | undefined; + try { + fd = binding.openBeneath( + parent.fd, + basename, + nativeOpenFlags( + fsSync.constants.O_WRONLY | fsSync.constants.O_CREAT | fsSync.constants.O_EXCL, + ), + ); + fsSync.fchmodSync(fd, mode); + created = fsSync.fstatSync(fd); + return wrapNativeFd(fd); + } catch (error) { + if (fd !== undefined) { + try { + fsSync.closeSync(fd); + } catch { + // Preserve the original error. + } + removeNativeCreatedFileIfStillPinned({ + binding, + parentPath, + parentFd: parent.fd, + basename, + created, + }); + } + throw error; + } finally { + await parent.close().catch(() => undefined); + } +} + +export function syncNativeFileBestEffort(fd: number): void { + try { + fsSync.fsyncSync(fd); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EPERM") { + throw error; + } + } +} + +export function writeNativeFd(fd: number, data: Buffer): void { + writeAll(fd, data); +} diff --git a/src/native-pinned-write.ts b/src/native-pinned-write.ts new file mode 100644 index 0000000..4ffa2b2 --- /dev/null +++ b/src/native-pinned-write.ts @@ -0,0 +1,157 @@ +import { randomUUID } from "node:crypto"; +import fsSync, { type Stats } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { FsSafeError } from "./errors.js"; +import type { FileIdentityStat } from "./file-identity.js"; +import { + removeNativeCreatedFileIfStillPinned, + syncNativeFileBestEffort, + writeNativeFd, +} from "./native-operations.js"; +import type { NativeBinding } from "./native.js"; +import type { PinnedWriteInput, PinnedWriteParams } from "./pinned-write.js"; + +function assertWithinMaxBytes(bytes: number, maxBytes: number | undefined): void { + if (maxBytes !== undefined && bytes > maxBytes) { + throw new FsSafeError( + "too-large", + `file exceeds limit of ${maxBytes} bytes (got at least ${bytes})`, + ); + } +} + +async function inputToBuffer( + input: PinnedWriteInput, + maxBytes: number | undefined, +): Promise { + if (input.kind === "buffer") { + const data = typeof input.data === "string" + ? Buffer.from(input.data, input.encoding ?? "utf8") + : Buffer.from(input.data); + assertWithinMaxBytes(data.byteLength, maxBytes); + return data; + } + const chunks: Buffer[] = []; + let bytes = 0; + for await (const chunk of input.stream) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array); + bytes += buffer.byteLength; + assertWithinMaxBytes(bytes, maxBytes); + chunks.push(buffer); + } + return Buffer.concat(chunks, bytes); +} + +function nativeOpenFlags(flags: number): number { + const closeOnExec = (fsSync.constants as typeof fsSync.constants & { O_CLOEXEC?: number }).O_CLOEXEC; + return ( + flags | + (closeOnExec ?? 0) | + (typeof fsSync.constants.O_NOFOLLOW === "number" ? fsSync.constants.O_NOFOLLOW : 0) + ); +} + +function sameNativeIdentity( + left: Pick, + right: Pick, +): boolean { + return left.dev === right.dev && left.ino === right.ino; +} + +export async function runPinnedWriteNative( + binding: NativeBinding, + params: PinnedWriteParams, +): Promise { + const data = await inputToBuffer(params.input, params.maxBytes); + const root = await fs.open( + params.rootPath, + fsSync.constants.O_RDONLY | + (typeof fsSync.constants.O_DIRECTORY === "number" ? fsSync.constants.O_DIRECTORY : 0), + ); + let parentFd: number | undefined; + let tempFd: number | undefined; + let targetFd: number | undefined; + let tempIdentity: Stats | undefined; + let parentPath = params.rootPath; + const tempName = `.${params.basename}.${randomUUID()}.native.tmp`; + let renamed = false; + try { + const rootIdentity = binding.fstatIdentity(root.fd); + if (params.rootIdentity && !sameNativeIdentity(params.rootIdentity, rootIdentity)) { + throw new FsSafeError("path-mismatch", "root path changed during native write"); + } + if (params.mkdir) { + binding.mkdirBeneath(root.fd, params.relativeParentPath, 0o777); + } + const parentFlags = + fsSync.constants.O_RDONLY | + (typeof fsSync.constants.O_DIRECTORY === "number" ? fsSync.constants.O_DIRECTORY : 0); + parentFd = binding.openBeneath(root.fd, params.relativeParentPath, parentFlags); + parentPath = await fs.realpath( + params.relativeParentPath + ? path.join(params.rootPath, ...params.relativeParentPath.split("/")) + : params.rootPath, + ); + const parentPathStat = await fs.lstat(parentPath); + const parentIdentity = binding.fstatIdentity(parentFd); + if ( + parentPathStat.isSymbolicLink() || + !sameNativeIdentity(parentPathStat, parentIdentity) + ) { + throw new FsSafeError("path-mismatch", "native write parent changed during resolution"); + } + try { + await fs.lstat(path.join(parentPath, params.basename)); + throw Object.assign(new Error("destination already exists"), { code: "EEXIST" }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + } + tempFd = binding.openBeneath( + parentFd, + tempName, + nativeOpenFlags( + fsSync.constants.O_WRONLY | fsSync.constants.O_CREAT | fsSync.constants.O_EXCL, + ), + ); + fsSync.fchmodSync(tempFd, params.mode || 0o600); + writeNativeFd(tempFd, data); + syncNativeFileBestEffort(tempFd); + tempIdentity = fsSync.fstatSync(tempFd); + binding.renameNoReplace(parentFd, tempName, parentFd, params.basename); + renamed = true; + targetFd = binding.openBeneath( + parentFd, + params.basename, + nativeOpenFlags(fsSync.constants.O_RDONLY), + ); + const targetIdentity = binding.fstatIdentity(targetFd); + if (!targetIdentity.isFile || !sameNativeIdentity(tempIdentity, targetIdentity)) { + throw new FsSafeError("path-mismatch", "native write target changed after rename"); + } + syncNativeFileBestEffort(parentFd); + return { dev: targetIdentity.dev, ino: targetIdentity.ino }; + } finally { + if (targetFd !== undefined) { + fsSync.closeSync(targetFd); + } + if (tempFd !== undefined) { + fsSync.closeSync(tempFd); + } + if (!renamed && parentFd !== undefined) { + removeNativeCreatedFileIfStillPinned({ + binding, + parentPath, + parentFd, + basename: tempName, + created: tempIdentity, + }); + } + if (parentFd !== undefined) { + fsSync.closeSync(parentFd); + } + await root.close().catch(() => undefined); + } +} diff --git a/src/native.ts b/src/native.ts new file mode 100644 index 0000000..690164b --- /dev/null +++ b/src/native.ts @@ -0,0 +1,59 @@ +import { createRequire } from "node:module"; +import { FsSafeError } from "./errors.js"; +import { getFsSafeNativeConfig } from "./native-config.js"; + +export type NativeBinding = typeof import("@openclaw/fs-safe-native"); + +const require = createRequire(import.meta.url); +let binding: NativeBinding | undefined; +let loadError: unknown; +let attempted = false; +let loadBinding = (): NativeBinding => require("@openclaw/fs-safe-native") as NativeBinding; + +export function getNativeBinding(): NativeBinding | undefined { + const { mode } = getFsSafeNativeConfig(); + if (mode === "off") { + return undefined; + } + if (!attempted) { + attempted = true; + try { + binding = loadBinding(); + } catch (error) { + loadError = error; + } + } + if (binding) { + return binding; + } + if (mode === "require") { + throw new FsSafeError("helper-unavailable", "native fs-safe helper is unavailable", { + cause: loadError, + }); + } + return undefined; +} + +export function requireNativeBinding(): NativeBinding { + const native = getNativeBinding(); + if (!native) { + throw new FsSafeError("helper-unavailable", "native fs-safe helper is unavailable", { + cause: loadError, + }); + } + return native; +} + +export function __setNativeLoaderForTest(loader: () => NativeBinding): void { + binding = undefined; + loadError = undefined; + attempted = false; + loadBinding = loader; +} + +export function __resetNativeLoaderForTest(): void { + binding = undefined; + loadError = undefined; + attempted = false; + loadBinding = () => require("@openclaw/fs-safe-native") as NativeBinding; +} diff --git a/src/owner-dacl.ts b/src/owner-dacl.ts new file mode 100644 index 0000000..6880f18 --- /dev/null +++ b/src/owner-dacl.ts @@ -0,0 +1,66 @@ +import { FsSafeError } from "./errors.js"; +import { getNativeBinding } from "./native.js"; + +export type WindowsAceFlags = { + raw: number; + objectInherit: boolean; + containerInherit: boolean; + noPropagateInherit: boolean; + inheritOnly: boolean; + inherited: boolean; + successfulAccess: boolean; + failedAccess: boolean; +}; + +export type WindowsAccessControlEntry = { + sid: string; + mask: number; + aceType: "allow" | "deny"; + flags: WindowsAceFlags; +}; + +export type OwnerAndDaclResult = + | { + status: "supported"; + ownerSid: string; + currentUserSid: string; + daclPresent: boolean; + isLocal: boolean; + complete: boolean; + unsupportedAceTypes: number[]; + aces: WindowsAccessControlEntry[]; + } + | { + status: "unsupported-platform"; + platform: NodeJS.Platform; + }; + +export function readOwnerAndDacl(targetPath: string): OwnerAndDaclResult { + if (process.platform !== "win32") { + return { status: "unsupported-platform", platform: process.platform }; + } + + const native = getNativeBinding(); + if (!native) { + throw new FsSafeError( + "helper-unavailable", + "Windows owner and DACL facts require the optional native binding", + ); + } + const facts = native.readOwnerAndDacl(targetPath); + return { + status: "supported", + ownerSid: facts.ownerSid, + currentUserSid: facts.currentUserSid, + daclPresent: facts.daclPresent, + isLocal: facts.isLocal, + complete: facts.aceListComplete, + unsupportedAceTypes: [...facts.unsupportedAceTypes], + aces: facts.aces.map((entry) => ({ + sid: entry.sid, + mask: entry.mask, + aceType: entry.aceType as "allow" | "deny", + flags: { ...entry.flags }, + })), + }; +} diff --git a/src/permissions-public.ts b/src/permissions-public.ts index a2400bf..0598244 100644 --- a/src/permissions-public.ts +++ b/src/permissions-public.ts @@ -13,3 +13,13 @@ export { type PermissionCheckOptions, type SafeStatResult, } from "./permissions.js"; +export { + createPrivateDirectory, + type CreatePrivateDirectoryOptions, +} from "./private-directory.js"; +export { + readOwnerAndDacl, + type OwnerAndDaclResult, + type WindowsAccessControlEntry, + type WindowsAceFlags, +} from "./owner-dacl.js"; diff --git a/src/permissions.ts b/src/permissions.ts index ee6893b..f050bd6 100644 --- a/src/permissions.ts +++ b/src/permissions.ts @@ -3,17 +3,15 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; - import { normalizeLowercaseStringOrEmpty } from "./string-coerce.js"; +import { inspectWindowsPermissionsNative } from "./windows-permissions-native.js"; import { resolveWindowsSystemCommand } from "./windows-command.js"; import { inspectWindowsOwner, resolveWindowsCurrentUserSid, resolveWindowsPrincipalSids, } from "./windows-owner.js"; - const execFileAsync = promisify(execFile); - export type PermissionExec = ( command: string, args: string[], @@ -186,6 +184,10 @@ export async function inspectPathPermissions( const bits = modeBits(effectiveMode); const platform = opts?.platform ?? process.platform; if (platform === "win32") { + const native = inspectWindowsPermissionsNative({ + targetPath, stat: st, effectiveIsDir, effectiveMode, bits, + }); + if (native) return native; const owner = await inspectWindowsOwner({ targetPath, env: opts?.env, diff --git a/src/pinned-helper.ts b/src/pinned-helper.ts deleted file mode 100644 index 5871d96..0000000 --- a/src/pinned-helper.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { DirEntry, PathStat } from "./types.js"; -import { - isPinnedHelperUnavailable, - runPinnedPythonOperation, - validatePinnedOperationPayload, -} from "./pinned-python.js"; - -type HelperOperation = "stat" | "readdir" | "mkdirp" | "remove" | "rename"; - -export { isPinnedHelperUnavailable }; - -export async function runPinnedHelper( - operation: HelperOperation, - rootDir: string, - payload: Record, -): Promise { - validatePinnedOperationPayload(payload); - return await runPinnedPythonOperation({ - operation, - rootPath: rootDir, - payload, - }); -} - -export async function helperStat(rootDir: string, relativePath: string): Promise { - return await runPinnedHelper("stat", rootDir, { relativePath }); -} - -export async function helperReaddir( - rootDir: string, - relativePath: string, - withFileTypes: false, -): Promise; -export async function helperReaddir( - rootDir: string, - relativePath: string, - withFileTypes: true, -): Promise; -export async function helperReaddir( - rootDir: string, - relativePath: string, - withFileTypes: boolean, -): Promise { - return await runPinnedHelper("readdir", rootDir, { - relativePath, - withFileTypes, - }); -} diff --git a/src/pinned-operation.ts b/src/pinned-operation.ts new file mode 100644 index 0000000..1a0eab5 --- /dev/null +++ b/src/pinned-operation.ts @@ -0,0 +1,39 @@ +import { FsSafeError } from "./errors.js"; + +export function validatePinnedOperationPayload(payload: Record): void { + if (typeof payload.relativePath === "string") { + validatePinnedRelativePath(payload.relativePath); + } + if (typeof payload.relativeParentPath === "string") { + validatePinnedRelativePath(payload.relativeParentPath); + } + if (typeof payload.from === "string") { + validatePinnedRelativePath(payload.from); + } + if (typeof payload.to === "string") { + validatePinnedRelativePath(payload.to); + } +} + +function validatePinnedRelativePath(relativePath: string): void { + if (relativePath.length === 0 || relativePath === ".") { + return; + } + if (relativePath.includes("\0")) { + throw new FsSafeError("invalid-path", "relative path contains a NUL byte"); + } + if ( + relativePath.startsWith("/") || + relativePath.startsWith("//") || + relativePath === ".." || + relativePath.startsWith("../") || + relativePath.startsWith("..\\") + ) { + throw new FsSafeError("invalid-path", "relative path must not escape root"); + } + for (const segment of relativePath.split("/")) { + if (segment === "..") { + throw new FsSafeError("invalid-path", "relative path must not contain '..'"); + } + } +} diff --git a/src/pinned-path.ts b/src/pinned-path.ts deleted file mode 100644 index 10d848d..0000000 --- a/src/pinned-path.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { FsSafeError } from "./errors.js"; -import { canFallbackFromPythonError } from "./pinned-python-config.js"; -import { runPinnedHelper } from "./pinned-helper.js"; - -export function isPinnedPathHelperSpawnError(error: unknown): boolean { - return canFallbackFromPythonError(error); -} - -export async function runPinnedPathHelper(params: { - operation: "mkdirp" | "remove"; - rootPath: string; - relativePath: string; -}): Promise { - try { - await runPinnedHelper(params.operation, params.rootPath, { - relativePath: params.relativePath, - }); - } catch (error) { - if (error instanceof FsSafeError) { - throw error; - } - throw new FsSafeError("helper-failed", "pinned path helper failed", { - cause: error instanceof Error ? error : undefined, - }); - } -} diff --git a/src/pinned-python-config.ts b/src/pinned-python-config.ts deleted file mode 100644 index 1325aa5..0000000 --- a/src/pinned-python-config.ts +++ /dev/null @@ -1,53 +0,0 @@ -export type FsSafePythonMode = "auto" | "off" | "require"; - -export type FsSafePythonConfig = { - mode: FsSafePythonMode; - pythonPath?: string; -}; - -let overrideConfig: Partial = {}; - -function parseMode(value: string | undefined): FsSafePythonMode | undefined { - if (!value) { - return undefined; - } - const normalized = value.trim().toLowerCase(); - if (normalized === "0" || normalized === "false" || normalized === "off" || normalized === "never") { - return "off"; - } - if (normalized === "1" || normalized === "true" || normalized === "on" || normalized === "auto") { - return "auto"; - } - if (normalized === "required" || normalized === "require") { - return "require"; - } - return undefined; -} - -export function configureFsSafePython(config: Partial): void { - overrideConfig = { ...overrideConfig, ...config }; -} - -export function getFsSafePythonConfig(): FsSafePythonConfig { - return { - mode: - overrideConfig.mode ?? - parseMode(process.env.FS_SAFE_PYTHON_MODE) ?? - parseMode(process.env.OPENCLAW_FS_SAFE_PYTHON_MODE) ?? - "auto", - pythonPath: - overrideConfig.pythonPath ?? - process.env.FS_SAFE_PYTHON ?? - process.env.OPENCLAW_FS_SAFE_PYTHON ?? - process.env.OPENCLAW_PINNED_PYTHON ?? - process.env.OPENCLAW_PINNED_WRITE_PYTHON, - }; -} - -export function canFallbackFromPythonError(error: unknown): boolean { - const code = error instanceof Error && "code" in error ? (error as { code?: unknown }).code : undefined; - return ( - getFsSafePythonConfig().mode !== "require" && - (code === "helper-unavailable" || code === "unsupported-platform") - ); -} diff --git a/src/pinned-python.ts b/src/pinned-python.ts deleted file mode 100644 index ff2f403..0000000 --- a/src/pinned-python.ts +++ /dev/null @@ -1,749 +0,0 @@ -import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; -import fsSync from "node:fs"; -import { FsSafeError } from "./errors.js"; -import { getFsSafePythonConfig } from "./pinned-python-config.js"; - -const PINNED_PYTHON_WORKER_SOURCE = String.raw` -import base64, errno, json, os, secrets, stat, sys -DIR_FLAGS = os.O_RDONLY -if hasattr(os, "O_DIRECTORY"): DIR_FLAGS |= os.O_DIRECTORY -if hasattr(os, "O_NOFOLLOW"): DIR_FLAGS |= os.O_NOFOLLOW -READ_FLAGS = os.O_RDONLY -if hasattr(os, "O_NONBLOCK"): READ_FLAGS |= os.O_NONBLOCK -if hasattr(os, "O_NOFOLLOW"): READ_FLAGS |= os.O_NOFOLLOW -WRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL -if hasattr(os, "O_NOFOLLOW"): - WRITE_FLAGS |= os.O_NOFOLLOW -def split_relative(value): - if value in ("", "."): - return [] - if "\x00" in value or value.startswith("/") or value.startswith("//"): - raise OSError(errno.EPERM, "invalid relative path") - if value.startswith("..\\"): - raise OSError(errno.EPERM, "path traversal is not allowed") - parts = [part for part in value.split("/") if part and part != "."] - for part in parts: - if part == "..": - raise OSError(errno.EPERM, "path traversal is not allowed") - return parts -def open_dir(path_value, dir_fd=None): - return os.open(path_value, DIR_FLAGS, dir_fd=dir_fd) -def walk_dir(root_fd, segments, mkdir_enabled=False): - current_fd = os.dup(root_fd) - try: - for segment in segments: - try: - next_fd = open_dir(segment, dir_fd=current_fd) - except FileNotFoundError: - if not mkdir_enabled: - raise - os.mkdir(segment, 0o777, dir_fd=current_fd) - next_fd = open_dir(segment, dir_fd=current_fd) - os.close(current_fd) - current_fd = next_fd - return current_fd - except Exception: - os.close(current_fd) - raise -def parent_and_basename(root_fd, relative): - segments = split_relative(relative) - if not segments: - raise OSError(errno.EPERM, "operation requires a non-root path") - parent_fd = walk_dir(root_fd, segments[:-1]) - return parent_fd, segments[-1] -def encode_stat(st): - mode = st.st_mode - return { - "dev": st.st_dev, - "gid": st.st_gid, - "ino": st.st_ino, - "isDirectory": stat.S_ISDIR(mode), - "isFile": stat.S_ISREG(mode), - "isSymbolicLink": stat.S_ISLNK(mode), - "mode": mode, - "mtimeMs": st.st_mtime * 1000, - "nlink": st.st_nlink, - "size": st.st_size, - "uid": st.st_uid, - } -def reject_unsafe_endpoint(st): - mode = st.st_mode - if stat.S_ISLNK(mode): - raise OSError(errno.ELOOP, "symlink endpoint is not allowed") - if stat.S_ISREG(mode) and st.st_nlink > 1: - raise OSError(errno.EPERM, "hardlinked file endpoint is not allowed") -def copy_bytes(source_fd, dest_fd): - while True: - chunk = os.read(source_fd, 65536) - if not chunk: - break - view = memoryview(chunk) - while view: - written = os.write(dest_fd, view) - if written <= 0: - raise OSError(errno.EIO, "short write") - view = view[written:] -def write_all(fd, data): - view = memoryview(data) - while view: - written = os.write(fd, view) - if written <= 0: - raise OSError(errno.EIO, "short write") - view = view[written:] -def fsync_best_effort(fd): - try: os.fsync(fd) - except OSError as error: - if error.errno != errno.EPERM: raise -def link_unsupported(exc): - unsupported = (errno.EPERM, errno.EOPNOTSUPP, getattr(errno, "ENOTSUP", errno.EOPNOTSUPP)) - return getattr(exc, "errno", None) in unsupported -def link_no_replace(name, new_name, source_fd, target_fd): - linked = False - try: - os.link(name, new_name, src_dir_fd=source_fd, dst_dir_fd=target_fd, follow_symlinks=False) - linked = True - os.unlink(name, dir_fd=source_fd) - except Exception: - if linked: - try: os.unlink(new_name, dir_fd=target_fd) - except FileNotFoundError: pass - raise - os.fsync(source_fd) - if source_fd != target_fd: - os.fsync(target_fd) -def copy_file_no_replace(source_parent_fd, source_name, target_parent_fd, basename, mode, expected=None, unlink_source=False): - source_fd = os.open(source_name, READ_FLAGS, dir_fd=source_parent_fd) - dest_fd = None; success = False; dest_stat = None - try: - if expected is not None: - source_stat = os.fstat(source_fd) - if source_stat.st_dev != expected.st_dev or source_stat.st_ino != expected.st_ino: - raise RuntimeError("fs-safe-source-mismatch") - dest_fd = os.open(basename, WRITE_FLAGS, mode, dir_fd=target_parent_fd) - copy_bytes(source_fd, dest_fd) - os.fsync(dest_fd) - dest_stat = os.fstat(dest_fd) - success = True - finally: - os.close(source_fd) - if dest_fd is not None: - os.close(dest_fd) - if dest_fd is not None and not success: - try: os.unlink(basename, dir_fd=target_parent_fd) - except FileNotFoundError: pass - if unlink_source: - try: - os.unlink(source_name, dir_fd=source_parent_fd) - except Exception: - try: os.unlink(basename, dir_fd=target_parent_fd) - except FileNotFoundError: pass - raise - return dest_stat -def same_identity(left, right): - return left.st_dev == right.st_dev and left.st_ino == right.st_ino -def verify_temp_name(parent_fd, temp_name, expected_stat): - current_stat = os.lstat(temp_name, dir_fd=parent_fd) - if stat.S_ISLNK(current_stat.st_mode) or not same_identity(current_stat, expected_stat): - raise RuntimeError("fs-safe-temp-mismatch") -def verify_committed_temp(parent_fd, basename, expected_stat): - final_stat = os.lstat(basename, dir_fd=parent_fd) - if not stat.S_ISLNK(final_stat.st_mode) and same_identity(final_stat, expected_stat): - return final_stat - try: os.unlink(basename, dir_fd=parent_fd) - except FileNotFoundError: pass - raise RuntimeError("fs-safe-temp-mismatch") -def commit_temp_file(parent_fd, temp_name, basename, overwrite, mode, expected_stat): - verify_temp_name(parent_fd, temp_name, expected_stat) - if overwrite: - os.replace(temp_name, basename, src_dir_fd=parent_fd, dst_dir_fd=parent_fd) - return verify_committed_temp(parent_fd, basename, expected_stat) - else: - try: - os.link(temp_name, basename, src_dir_fd=parent_fd, dst_dir_fd=parent_fd, follow_symlinks=False) - final_stat = verify_committed_temp(parent_fd, basename, expected_stat) - os.unlink(temp_name, dir_fd=parent_fd) - return final_stat - except OSError as exc: - if not link_unsupported(exc): - raise - return copy_file_no_replace(parent_fd, temp_name, parent_fd, basename, mode, expected_stat, True) -def assert_expected_root(root_fd, payload): - if "rootDev" in payload or "rootIno" in payload: - root_stat = os.fstat(root_fd) - if root_stat.st_dev != int(payload["rootDev"]) or root_stat.st_ino != int(payload["rootIno"]): - raise RuntimeError("fs-safe-root-mismatch") -def stat_path(root_fd, payload): - relative = payload.get("relativePath", "") - segments = split_relative(relative) - if not segments: - return encode_stat(os.fstat(root_fd)) - parent_fd, basename = parent_and_basename(root_fd, relative) - try: - st = os.lstat(basename, dir_fd=parent_fd) - if payload.get("rejectSymlink", True) and stat.S_ISLNK(st.st_mode): - raise OSError(errno.ELOOP, "symlink endpoint is not allowed") - return encode_stat(st) - finally: - os.close(parent_fd) -def readdir_path(root_fd, payload): - dir_fd = walk_dir(root_fd, split_relative(payload.get("relativePath", ""))) - try: - names = sorted(os.listdir(dir_fd)) - if not payload.get("withFileTypes", False): - return names - entries = [] - for name in names: - st = os.lstat(name, dir_fd=dir_fd) - entry = encode_stat(st) - entry["name"] = name - entries.append(entry) - return entries - finally: - os.close(dir_fd) -def mkdirp_path(root_fd, payload): - dir_fd = walk_dir(root_fd, split_relative(payload.get("relativePath", "")), mkdir_enabled=True) - os.close(dir_fd); return None -def remove_tree(parent_fd, basename): - st = os.lstat(basename, dir_fd=parent_fd) - if stat.S_ISDIR(st.st_mode) and not stat.S_ISLNK(st.st_mode): - dir_fd = open_dir(basename, dir_fd=parent_fd) - try: - for child in os.listdir(dir_fd): - remove_tree(dir_fd, child) - finally: - os.close(dir_fd) - os.rmdir(basename, dir_fd=parent_fd) - else: - os.unlink(basename, dir_fd=parent_fd) -def remove_path(root_fd, payload): - parent_fd, basename = parent_and_basename(root_fd, payload.get("relativePath", "")) - try: - try: - st = os.lstat(basename, dir_fd=parent_fd) - except FileNotFoundError: - if payload.get("force", True): - return None - raise - if stat.S_ISDIR(st.st_mode) and not stat.S_ISLNK(st.st_mode): - if payload.get("recursive", False): - remove_tree(parent_fd, basename) - else: - os.rmdir(basename, dir_fd=parent_fd) - else: - os.unlink(basename, dir_fd=parent_fd) - return None - finally: - os.close(parent_fd) - -def rename_path(root_fd, payload): - from_parent_fd, from_base = parent_and_basename(root_fd, payload["from"]) - to_parent_fd, to_base = parent_and_basename(root_fd, payload["to"]) - try: - from_stat = os.lstat(from_base, dir_fd=from_parent_fd) - reject_unsafe_endpoint(from_stat) - overwrite = payload.get("overwrite", True) - if not overwrite and stat.S_ISREG(from_stat.st_mode): - try: - link_no_replace(from_base, to_base, from_parent_fd, to_parent_fd) - except OSError as exc: - if not link_unsupported(exc): - raise - copy_file_no_replace(from_parent_fd, from_base, to_parent_fd, to_base, stat.S_IMODE(from_stat.st_mode), from_stat, True) - return None - if not overwrite and stat.S_ISDIR(from_stat.st_mode): - raise RuntimeError("fs-safe-directory-noreplace-unsupported") - if not overwrite: - try: - os.lstat(to_base, dir_fd=to_parent_fd) - raise FileExistsError(errno.EEXIST, "destination exists", to_base) - except FileNotFoundError: - pass - os.rename(from_base, to_base, src_dir_fd=from_parent_fd, dst_dir_fd=to_parent_fd) - os.fsync(from_parent_fd) - if from_parent_fd != to_parent_fd: - os.fsync(to_parent_fd) - return None - finally: - os.close(from_parent_fd) - os.close(to_parent_fd) - -def create_temp_file(parent_fd, basename, mode): - prefix = "." + basename + "." - for _ in range(128): - candidate = prefix + secrets.token_hex(6) + ".tmp" - try: - fd = os.open(candidate, WRITE_FLAGS, mode, dir_fd=parent_fd) - return candidate, fd - except FileExistsError: - continue - raise RuntimeError("failed to allocate pinned temp file") - -def write_path(root_fd, payload): - parent_fd = walk_dir(root_fd, split_relative(payload.get("relativeParentPath", "")), bool(payload.get("mkdir", True))) - temp_fd = None - temp_name = None - basename = payload["basename"] - mode = int(payload.get("mode", 0o600)) - overwrite = bool(payload.get("overwrite", True)) - max_bytes = int(payload.get("maxBytes", -1)) - data = base64.b64decode(payload.get("base64", "")) - try: - if max_bytes >= 0 and len(data) > max_bytes: - raise RuntimeError("fs-safe-too-large:%d:%d" % (max_bytes, len(data))) - if not overwrite: - try: - os.lstat(basename, dir_fd=parent_fd) - raise FileExistsError(errno.EEXIST, "destination exists", basename) - except FileNotFoundError: - pass - temp_name, temp_fd = create_temp_file(parent_fd, basename, mode) - os.fchmod(temp_fd, mode) - write_all(temp_fd, data) - fsync_best_effort(temp_fd) - temp_stat = os.fstat(temp_fd) - os.close(temp_fd) - temp_fd = None - result_stat = commit_temp_file(parent_fd, temp_name, basename, overwrite, mode, temp_stat) - temp_name = None - fsync_best_effort(parent_fd) - return {"dev": result_stat.st_dev, "ino": result_stat.st_ino} - finally: - if temp_fd is not None: - os.close(temp_fd) - if temp_name is not None: - try: - os.unlink(temp_name, dir_fd=parent_fd) - except FileNotFoundError: - pass - os.close(parent_fd) - -def copy_path(root_fd, payload): - source_fd = os.open(payload["sourcePath"], READ_FLAGS) - parent_fd = None - temp_fd = None - temp_name = None - try: - source_stat = os.fstat(source_fd) - if not stat.S_ISREG(source_stat.st_mode): - raise RuntimeError("fs-safe-not-file") - if source_stat.st_dev != int(payload["sourceDev"]) or source_stat.st_ino != int(payload["sourceIno"]): - raise RuntimeError("fs-safe-source-mismatch") - basename = payload["basename"] - mode = int(payload.get("mode", 0o600)) - overwrite = bool(payload.get("overwrite", True)) - max_bytes = int(payload.get("maxBytes", -1)) - if max_bytes >= 0 and source_stat.st_size > max_bytes: - raise RuntimeError("fs-safe-too-large:%d:%d" % (max_bytes, source_stat.st_size)) - parent_fd = walk_dir(root_fd, split_relative(payload.get("relativeParentPath", "")), bool(payload.get("mkdir", True))) - temp_name, temp_fd = create_temp_file(parent_fd, basename, mode) - os.fchmod(temp_fd, mode) - written_bytes = 0 - while True: - chunk = os.read(source_fd, 65536) - if not chunk: - break - written_bytes += len(chunk) - if max_bytes >= 0 and written_bytes > max_bytes: - raise RuntimeError("fs-safe-too-large:%d:%d" % (max_bytes, written_bytes)) - view = memoryview(chunk) - while view: - written = os.write(temp_fd, view) - if written <= 0: - raise OSError(errno.EIO, "short write") - view = view[written:] - os.fsync(temp_fd) - temp_stat = os.fstat(temp_fd) - os.close(temp_fd) - temp_fd = None - result_stat = commit_temp_file(parent_fd, temp_name, basename, overwrite, mode, temp_stat) - temp_name = None - os.fsync(parent_fd) - return {"dev": result_stat.st_dev, "ino": result_stat.st_ino} - finally: - os.close(source_fd) - if temp_fd is not None: - os.close(temp_fd) - if temp_name is not None and parent_fd is not None: - try: - os.unlink(temp_name, dir_fd=parent_fd) - except FileNotFoundError: - pass - if parent_fd is not None: - os.close(parent_fd) - -def run_operation(operation, root_path, payload): - root_fd = open_dir(root_path) - try: - assert_expected_root(root_fd, payload) - if operation == "stat": - return stat_path(root_fd, payload) - if operation == "readdir": - return readdir_path(root_fd, payload) - if operation == "mkdirp": - return mkdirp_path(root_fd, payload) - if operation == "remove": - return remove_path(root_fd, payload) - if operation == "rename": - return rename_path(root_fd, payload) - if operation == "write": - return write_path(root_fd, payload) - if operation == "copy": - return copy_path(root_fd, payload) - raise RuntimeError("unknown operation: " + operation) - finally: - os.close(root_fd) - -for line in sys.stdin: - try: - request = json.loads(line) - result = run_operation(request["operation"], request["rootPath"], request.get("payload") or {}) - response = {"id": request["id"], "ok": True, "result": result} - except Exception as exc: - response = { - "id": request.get("id") if isinstance(locals().get("request"), dict) else None, - "ok": False, - "code": exc.__class__.__name__, - "errno": getattr(exc, "errno", None), - "message": str(exc), - } - print(json.dumps(response, separators=(",", ":")), flush=True) -`; - -type PinnedPythonOperation = - | "copy" - | "stat" - | "readdir" - | "mkdirp" - | "remove" - | "rename" - | "write"; - -type PendingRequest = { - reject(error: unknown): void; - resolve(value: unknown): void; -}; - -type PinnedPythonWorker = { - child: ChildProcessWithoutNullStreams; - pending: Map; - stderr: string; - stdoutBuffer: string; -}; - -let nextRequestId = 1; -let worker: PinnedPythonWorker | null = null; - -export function __resetPinnedPythonWorkerForTest(): void { - const currentWorker = worker; - worker = null; - if (!currentWorker) { - return; - } - currentWorker.pending.clear(); - currentWorker.child.kill("SIGTERM"); -} - -const PYTHON_CANDIDATE_DEFAULTS = [ - "/usr/bin/python3", - "/opt/homebrew/bin/python3", - "/usr/local/bin/python3", -]; - -function canExecute(binPath: string): boolean { - try { - fsSync.accessSync(binPath, fsSync.constants.X_OK); - return true; - } catch { - return false; - } -} - -function resolvePython(): string { - const configured = getFsSafePythonConfig().pythonPath; - if (configured) { - return configured; - } - for (const candidate of PYTHON_CANDIDATE_DEFAULTS) { - if (canExecute(candidate)) { - return candidate; - } - } - return "python3"; -} - -function assertPinnedHelperSupported(): void { - if (process.platform === "win32") { - throw new FsSafeError( - "unsupported-platform", - "fd-relative pinned filesystem operations are not available on Windows", - ); - } - if (getFsSafePythonConfig().mode === "off") { - throw new FsSafeError("helper-unavailable", "Python helper is disabled"); - } -} - -function isSpawnUnavailable(error: unknown): boolean { - if (!(error instanceof Error)) { - return false; - } - const maybeErrno = error as NodeJS.ErrnoException; - return ( - typeof maybeErrno.syscall === "string" && - maybeErrno.syscall.startsWith("spawn") && - ["EACCES", "ENOENT", "ENOEXEC"].includes(maybeErrno.code ?? "") - ); -} - -function mapWorkerError(response: Record): Error { - const code = typeof response.code === "string" ? response.code : ""; - const errno = typeof response.errno === "number" ? response.errno : undefined; - const message = - typeof response.message === "string" && response.message - ? response.message - : "pinned helper failed"; - const tooLarge = message.match(/fs-safe-too-large:(\d+):(\d+)/); - if (tooLarge) { - const [, limit, got] = tooLarge; - return new FsSafeError("too-large", `file exceeds limit of ${limit} bytes (got at least ${got})`); - } - if (message.includes("fs-safe-not-file")) { - return new FsSafeError("not-file", "not a file"); - } - if (message.includes("fs-safe-source-mismatch")) { - return new FsSafeError("path-mismatch", "source path changed during copy"); - } - if (message.includes("fs-safe-temp-mismatch")) { - return new FsSafeError("path-mismatch", "temp path changed during write"); - } - if (message.includes("fs-safe-root-mismatch")) { - return new FsSafeError("path-mismatch", "root path changed during operation"); - } - if (message.includes("fs-safe-directory-noreplace-unsupported")) { - return new FsSafeError("invalid-path", "directory moves require overwrite: true"); - } - if (code === "FileNotFoundError" || errno === 2) { - return new FsSafeError("not-found", "file not found"); - } - if (code === "FileExistsError" || errno === 17) { - return new FsSafeError("already-exists", message); - } - if (errno === 39) { - return new FsSafeError("not-empty", "directory is not empty"); - } - if (errno === 1 || errno === 13 || errno === 21) { - return new FsSafeError("not-removable", "path is not removable under root"); - } - if (code === "NotADirectoryError" || code === "OSError" || errno === 20 || errno === 40) { - return new FsSafeError("path-alias", message); - } - return new FsSafeError("helper-failed", message); -} - -function rejectPending(error: Error, targetWorker = worker): void { - if (!targetWorker || worker !== targetWorker) { - return; - } - setWorkerRef(targetWorker, false); - for (const pending of targetWorker.pending.values()) { - pending.reject(error); - } - targetWorker.pending.clear(); - worker = null; -} - -function handleWorkerLine(currentWorker: PinnedPythonWorker, line: string): void { - if (worker !== currentWorker || !line.trim()) { - return; - } - let decoded: unknown; - try { - decoded = JSON.parse(line) as unknown; - } catch { - rejectPending( - new FsSafeError("helper-failed", `pinned helper returned invalid JSON: ${line}`), - currentWorker, - ); - return; - } - if (typeof decoded !== "object" || decoded === null || !("id" in decoded)) { - rejectPending( - new FsSafeError("helper-failed", "pinned helper returned invalid response"), - currentWorker, - ); - return; - } - const response = decoded as { id?: unknown; ok?: unknown; result?: unknown }; - const id = typeof response.id === "number" ? response.id : undefined; - if (id === undefined) { - return; - } - const pending = currentWorker.pending.get(id); - if (!pending) { - return; - } - currentWorker.pending.delete(id); - if (currentWorker.pending.size === 0) { - setWorkerRef(currentWorker, false); - } - if (response.ok === true) { - pending.resolve(response.result); - return; - } - pending.reject(mapWorkerError(decoded as Record)); -} - -function getWorker() { - assertPinnedHelperSupported(); - if (worker) { - return worker; - } - const child = spawn(resolvePython(), ["-u", "-c", PINNED_PYTHON_WORKER_SOURCE], { - stdio: ["pipe", "pipe", "pipe"], - }); - const currentWorker: PinnedPythonWorker = { child, pending: new Map(), stderr: "", stdoutBuffer: "" }; - worker = currentWorker; - child.stdout.setEncoding("utf8"); - child.stderr.setEncoding("utf8"); - child.stdout.on("data", (chunk: string) => { - if (worker !== currentWorker) { - return; - } - currentWorker.stdoutBuffer += chunk; - for (;;) { - const newline = currentWorker.stdoutBuffer.indexOf("\n"); - if (newline < 0) { - break; - } - const line = currentWorker.stdoutBuffer.slice(0, newline); - currentWorker.stdoutBuffer = currentWorker.stdoutBuffer.slice(newline + 1); - handleWorkerLine(currentWorker, line); - } - }); - child.stderr.on("data", (chunk: string) => { - if (worker === currentWorker) { - currentWorker.stderr = `${currentWorker.stderr}${chunk}`.slice(-4096); - } - }); - child.once("error", (error) => { - const mapped = isSpawnUnavailable(error) - ? new FsSafeError("helper-unavailable", "Python helper is unavailable", { cause: error }) - : error instanceof Error - ? error - : new Error(String(error)); - rejectPending(mapped, currentWorker); - }); - child.once("close", (code, signal) => { - const stderr = currentWorker.stderr.trim(); - rejectPending( - new FsSafeError( - "helper-failed", - stderr || `pinned helper exited with code ${code ?? "null"} (${signal ?? "?"})`, - ), - currentWorker, - ); - }); - process.once("exit", () => { - child.kill("SIGTERM"); - }); - setWorkerRef(currentWorker, false); - return currentWorker; -} - -function setRefable(value: unknown, ref: boolean): void { - if (!value) { - return; - } - const method = ref ? "ref" : "unref"; - const refable = value as { ref?: () => void; unref?: () => void }; - refable[method]?.(); -} - -function setWorkerRef(currentWorker: PinnedPythonWorker, ref: boolean): void { - setRefable(currentWorker.child, ref); - setRefable(currentWorker.child.stdin, ref); - setRefable(currentWorker.child.stdout, ref); - setRefable(currentWorker.child.stderr, ref); -} - -export async function runPinnedPythonOperation(params: { - operation: PinnedPythonOperation; - rootPath: string; - payload: Record; -}): Promise { - const requestId = nextRequestId++; - const currentWorker = getWorker(); - if (typeof currentWorker.child.stdin?.write !== "function") { - throw new FsSafeError("helper-unavailable", "Python helper stdin is unavailable"); - } - setWorkerRef(currentWorker, true); - return await new Promise((resolve, reject) => { - currentWorker.pending.set(requestId, { - reject, - resolve: (value) => resolve(value as T), - }); - const request = JSON.stringify({ - id: requestId, - operation: params.operation, - rootPath: params.rootPath, - payload: params.payload, - }); - currentWorker.child.stdin.write(`${request}\n`, (error) => { - if (error) { - currentWorker.pending.delete(requestId); - if (currentWorker.pending.size === 0) { - setWorkerRef(currentWorker, false); - } - reject(error); - } - }); - }); -} - -export function assertPinnedPythonOperationAvailable(): void { - const currentWorker = getWorker(); - if (typeof currentWorker.child.stdin?.write !== "function") { - throw new FsSafeError("helper-unavailable", "Python helper stdin is unavailable"); - } -} - -export function validatePinnedOperationPayload(payload: Record): void { - if (typeof payload.relativePath === "string") { - validatePinnedRelativePath(payload.relativePath); - } - if (typeof payload.relativeParentPath === "string") { - validatePinnedRelativePath(payload.relativeParentPath); - } - if (typeof payload.from === "string") { - validatePinnedRelativePath(payload.from); - } - if (typeof payload.to === "string") { - validatePinnedRelativePath(payload.to); - } -} - -export function isPinnedHelperUnavailable(error: unknown): boolean { - return error instanceof Error && "code" in error && (error as { code?: unknown }).code === "helper-unavailable"; -} -function validatePinnedRelativePath(relativePath: string): void { - if (relativePath.length === 0 || relativePath === ".") { - return; - } - if (relativePath.includes("\0")) { - throw new FsSafeError("invalid-path", "relative path contains a NUL byte"); - } - if ( - relativePath.startsWith("/") || - relativePath.startsWith("//") || - relativePath === ".." || - relativePath.startsWith("../") || - relativePath.startsWith("..\\") - ) { - throw new FsSafeError("invalid-path", "relative path must not escape root"); - } - for (const segment of relativePath.split("/")) { - if (segment === "..") { - throw new FsSafeError("invalid-path", "relative path must not contain '..'"); - } - } -} diff --git a/src/pinned-write.ts b/src/pinned-write.ts index bed52ab..db25b5c 100644 --- a/src/pinned-write.ts +++ b/src/pinned-write.ts @@ -11,16 +11,13 @@ import type { FileIdentityStat } from "./file-identity.js"; import { sameFileIdentity, sha256Hex } from "./file-identity.js"; import { withAsyncDirectoryGuards } from "./guarded-mutation.js"; import { mkdirPathComponentsWithGuards } from "./guarded-mkdir.js"; -import { canFallbackFromPythonError, getFsSafePythonConfig } from "./pinned-python-config.js"; -import { - assertPinnedPythonOperationAvailable, - runPinnedPythonOperation, - validatePinnedOperationPayload, -} from "./pinned-python.js"; +import { runPinnedWriteNative } from "./native-pinned-write.js"; +import { getNativeBinding } from "./native.js"; +import { validatePinnedOperationPayload } from "./pinned-operation.js"; import { withSidecarLock } from "./sidecar-lock.js"; import { getFsSafeTestHooks } from "./test-hooks.js"; -type PinnedWriteInput = +export type PinnedWriteInput = | { kind: "buffer"; data: string | Buffer; encoding?: BufferEncoding } | { kind: "stream"; stream: Readable }; @@ -86,34 +83,11 @@ async function writeStreamToHandle( } } -async function inputToBase64( - input: PinnedWriteInput, - maxBytes: number | undefined, -): Promise { - if (input.kind === "buffer") { - assertWithinMaxBytes(byteLength(input.data, input.encoding), maxBytes); - return ( - typeof input.data === "string" - ? Buffer.from(input.data, input.encoding ?? "utf8") - : input.data - ).toString("base64"); - } - const chunks: Buffer[] = []; - let bytes = 0; - for await (const chunk of input.stream) { - const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array); - bytes += buffer.byteLength; - assertWithinMaxBytes(bytes, maxBytes); - chunks.push(buffer); - } - return Buffer.concat(chunks, bytes).toString("base64"); -} - -type RenameIdentityMismatchPolicy = "throw" | "verify-content"; +export type RenameIdentityMismatchPolicy = "throw" | "verify-content"; export type RenameIdentityPolicy = "strict" | "verify-content-with-lock"; -type PinnedWriteParams = { +export type PinnedWriteParams = { rootPath: string; relativeParentPath: string; basename: string; @@ -131,51 +105,16 @@ export async function runPinnedWriteHelper(params: PinnedWriteParams): Promise({ - operation: "write", - rootPath: params.rootPath, - payload, - }); - } catch (error) { - if (canFallbackFromPythonError(error)) { - return await runPinnedWriteFallback({ ...params, input }); - } - throw error; + const native = getNativeBinding(); + if (native && params.overwrite === false) { + return await runPinnedWriteNative(native, params); } + return await runPinnedWriteFallback(params); } export async function runPinnedWriteWithRenamePolicy( @@ -212,40 +151,6 @@ export async function runPinnedWriteWithRenamePolicy( ); } -export async function runPinnedCopyHelper(params: { - rootPath: string; - relativeParentPath: string; - basename: string; - mkdir: boolean; - mode: number; - overwrite?: boolean; - maxBytes?: number; - sourcePath: string; - sourceIdentity: FileIdentityStat; - rootIdentity?: FileIdentityStat; -}): Promise { - assertSafeBasename(params.basename); - validatePinnedOperationPayload({ - relativeParentPath: params.relativeParentPath, - }); - return await runPinnedPythonOperation({ - operation: "copy", - rootPath: params.rootPath, - payload: { - basename: params.basename, - maxBytes: params.maxBytes ?? -1, - mkdir: params.mkdir, - mode: params.mode || 0o600, - overwrite: params.overwrite !== false, - relativeParentPath: params.relativeParentPath, - ...(params.rootIdentity ? { rootDev: params.rootIdentity.dev, rootIno: params.rootIdentity.ino } : {}), - sourceDev: params.sourceIdentity.dev, - sourceIno: params.sourceIdentity.ino, - sourcePath: params.sourcePath, - }, - }); -} - async function runPinnedWriteFallback(params: { rootPath: string; relativeParentPath: string; @@ -293,6 +198,7 @@ async function runPinnedWriteFallback(params: { ); let created = true; try { + await handle.chmod(params.mode); if (params.input.kind === "buffer") { assertWithinMaxBytes( byteLength(params.input.data, params.input.encoding), @@ -334,6 +240,7 @@ async function runPinnedWriteFallback(params: { let renamed = false; try { handle = await fs.open(tempPath, tempFlags, params.mode); + await handle.chmod(params.mode); if (params.input.kind === "buffer") { assertWithinMaxBytes( byteLength(params.input.data, params.input.encoding), diff --git a/src/private-directory.ts b/src/private-directory.ts new file mode 100644 index 0000000..16d7cf9 --- /dev/null +++ b/src/private-directory.ts @@ -0,0 +1,28 @@ +import { FsSafeError } from "./errors.js"; +import { getNativeBinding } from "./native.js"; + +export type CreatePrivateDirectoryOptions = { + platform?: NodeJS.Platform; +}; + +export async function createPrivateDirectory( + targetPath: string, + options?: CreatePrivateDirectoryOptions, +): Promise { + const platform = options?.platform ?? process.platform; + if (platform !== "win32") { + throw new FsSafeError( + "helper-unavailable", + "private-directory creation is available only on Windows through the optional native binding", + ); + } + + const native = getNativeBinding(); + if (!native) { + throw new FsSafeError( + "helper-unavailable", + "private Windows directory creation requires the optional native binding", + ); + } + native.createPrivateDirectory(targetPath); +} diff --git a/src/private-temp-workspace.ts b/src/private-temp-workspace.ts index cbe40be..45ddc1a 100644 --- a/src/private-temp-workspace.ts +++ b/src/private-temp-workspace.ts @@ -9,7 +9,13 @@ import { type FileStoreSync, } from "./file-store.js"; import { openRootFileSync } from "./root-file.js"; -import { registerTempPathForExit } from "./temp-cleanup.js"; +import { sameFileIdentity } from "./file-identity.js"; +import { + registerTempPathForExit, + type TempPathIdentityReceipt, +} from "./temp-cleanup.js"; + +export type TempWorkspaceCleanupResult = "removed" | "missing" | "identity-mismatch"; export type TempWorkspaceOptions = { rootDir: string; @@ -20,6 +26,7 @@ export type TempWorkspaceOptions = { export type TempWorkspace = { dir: string; + identity: TempPathIdentityReceipt; store: FileStore; path(fileName: string): string; write(fileName: string, data: string | Uint8Array): Promise; @@ -31,19 +38,20 @@ export type TempWorkspace = { ): Promise; copyIn(fileName: string, sourcePath: string): Promise; read(fileName: string): Promise; - cleanup(): Promise; + cleanup(): Promise; [Symbol.asyncDispose](): Promise; }; export type TempWorkspaceSync = { dir: string; + identity: TempPathIdentityReceipt; store: FileStoreSync; path(fileName: string): string; write(fileName: string, data: string | Uint8Array): string; writeText(fileName: string, data: string): string; writeJson(fileName: string, data: unknown, options?: { trailingNewline?: boolean }): string; read(fileName: string): Buffer; - cleanup(): void; + cleanup(): TempWorkspaceCleanupResult; [Symbol.dispose](): void; }; @@ -97,6 +105,40 @@ function ensurePrivateDirectorySync(dir: string, mode: number): void { } } +async function cleanupWorkspace( + dir: string, + identity: TempPathIdentityReceipt, +): Promise { + let current; + try { + current = await fs.lstat(dir); + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ENOENT" ? "missing" : "identity-mismatch"; + } + if (!sameFileIdentity(current, identity)) { + return "identity-mismatch"; + } + await fs.rm(dir, { recursive: true, force: true }); + return "removed"; +} + +function cleanupWorkspaceSync( + dir: string, + identity: TempPathIdentityReceipt, +): TempWorkspaceCleanupResult { + let current; + try { + current = fsSync.lstatSync(dir); + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ENOENT" ? "missing" : "identity-mismatch"; + } + if (!sameFileIdentity(current, identity)) { + return "identity-mismatch"; + } + fsSync.rmSync(dir, { recursive: true, force: true }); + return "removed"; +} + async function createTempWorkspace( options: TempWorkspaceOptions, ): Promise { @@ -106,16 +148,18 @@ async function createTempWorkspace( const root = await fs.realpath(requestedRoot).catch(() => requestedRoot); await ensurePrivateDirectory(root, dirMode); const dir = await fs.mkdtemp(path.join(root, sanitizeTempPrefix(options.prefix))); - const unregisterTempDir = registerTempPathForExit(dir, { recursive: true }); await fs.chmod(dir, dirMode).catch(() => undefined); const stat = await fs.lstat(dir); if (stat.isSymbolicLink() || !stat.isDirectory()) { throw new Error(`Temp workspace must be a directory: ${dir}`); } + const identity = { dev: stat.dev, ino: stat.ino }; + const unregisterTempDir = registerTempPathForExit(dir, { recursive: true, identity }); const store = fileStore({ rootDir: dir, private: true, dirMode, mode }); return { dir, + identity, store, path: (fileName) => resolveWorkspaceLeaf(dir, fileName), write: async (fileName, data) => @@ -132,14 +176,14 @@ async function createTempWorkspace( read: async (fileName) => await store.readBytes(assertWorkspaceFileName(fileName)), cleanup: async () => { try { - await fs.rm(dir, { recursive: true, force: true }).catch(() => undefined); + return await cleanupWorkspace(dir, identity); } finally { unregisterTempDir(); } }, [Symbol.asyncDispose]: async () => { try { - await fs.rm(dir, { recursive: true, force: true }).catch(() => undefined); + await cleanupWorkspace(dir, identity); } finally { unregisterTempDir(); } @@ -182,7 +226,6 @@ export function tempWorkspaceSync( } ensurePrivateDirectorySync(root, dirMode); const dir = fsSync.mkdtempSync(path.join(root, sanitizeTempPrefix(options.prefix))); - const unregisterTempDir = registerTempPathForExit(dir, { recursive: true }); try { fsSync.chmodSync(dir, dirMode); } catch { @@ -192,10 +235,13 @@ export function tempWorkspaceSync( if (stat.isSymbolicLink() || !stat.isDirectory()) { throw new Error(`Temp workspace must be a directory: ${dir}`); } + const identity = { dev: stat.dev, ino: stat.ino }; + const unregisterTempDir = registerTempPathForExit(dir, { recursive: true, identity }); const store = fileStoreSync({ rootDir: dir, private: true, dirMode, mode }); return { dir, + identity, store, path: (fileName) => resolveWorkspaceLeaf(dir, fileName), write: (fileName, data) => @@ -225,18 +271,14 @@ export function tempWorkspaceSync( }, cleanup: () => { try { - fsSync.rmSync(dir, { recursive: true, force: true }); - } catch { - // Best-effort cleanup. + return cleanupWorkspaceSync(dir, identity); } finally { unregisterTempDir(); } }, [Symbol.dispose]: () => { try { - fsSync.rmSync(dir, { recursive: true, force: true }); - } catch { - // Best-effort cleanup. + cleanupWorkspaceSync(dir, identity); } finally { unregisterTempDir(); } diff --git a/src/publish-file-failure.ts b/src/publish-file-failure.ts new file mode 100644 index 0000000..976c2ef --- /dev/null +++ b/src/publish-file-failure.ts @@ -0,0 +1,69 @@ +import { FsSafeError } from "./errors.js"; +import type { FileIdentityStat } from "./file-identity.js"; + +export type PublishFileExclusiveSyncFailurePolicy = "rollback" | "preserve"; +export type PublishFileExclusiveDirectorySyncFailure = { + status: "failed"; + code?: string; +}; +export type PublishFileExclusiveFailurePhase = + | "copy-create" + | "copy-verify" + | "directory-sync" + | "hardlink-create" + | "hardlink-verify" + | "rename-create" + | "rename-verify"; +export type PublishFileExclusiveCleanup = "removed" | "preserved" | "unknown"; +export type PublishFileExclusiveFailureDetails = { + phase: PublishFileExclusiveFailurePhase; + targetCreated: boolean; + targetIdentity?: FileIdentityStat; + cleanup: PublishFileExclusiveCleanup; + directorySync?: PublishFileExclusiveDirectorySyncFailure; +}; + +export type PublishFailureState = { + phase: PublishFileExclusiveFailurePhase; + targetCreated: boolean; + targetIdentity?: FileIdentityStat; + preserveTarget: boolean; + directorySync?: PublishFileExclusiveDirectorySyncFailure; +}; + +export function rememberCreatedTarget( + state: PublishFailureState, + identity: FileIdentityStat, + phase: PublishFileExclusiveFailurePhase, +): void { + state.targetCreated = true; + state.targetIdentity = { dev: identity.dev, ino: identity.ino }; + state.phase = phase; +} + +export function publicationFailure( + error: unknown, + state: PublishFailureState, + cleanup: PublishFileExclusiveCleanup, +): FsSafeError { + const cause = error instanceof Error ? error : new Error(String(error)); + const details: PublishFileExclusiveFailureDetails = { + phase: state.phase, + targetCreated: state.targetCreated, + ...(state.targetIdentity ? { targetIdentity: state.targetIdentity } : {}), + ...(state.directorySync ? { directorySync: state.directorySync } : {}), + cleanup, + }; + return new FsSafeError( + error instanceof FsSafeError ? error.code : "helper-failed", + `exclusive file publication failed during ${state.phase}: ${cause.message}`, + { cause, details }, + ); +} + +export function directorySyncFailure( + error: unknown, +): PublishFileExclusiveDirectorySyncFailure { + const code = (error as { code?: unknown } | undefined)?.code; + return typeof code === "string" ? { status: "failed", code } : { status: "failed" }; +} diff --git a/src/publish-file.ts b/src/publish-file.ts new file mode 100644 index 0000000..3ef49fc --- /dev/null +++ b/src/publish-file.ts @@ -0,0 +1,472 @@ +import { createHash } from "node:crypto"; +import fsSync, { type Stats } from "node:fs"; +import type { FileHandle } from "node:fs/promises"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { + pinDirectory, + type DirectoryReceipt, + type DirectorySyncOutcome, +} from "./directory-durability.js"; +import { FsSafeError } from "./errors.js"; +import { hashFileHandle } from "./file-hash.js"; +import { sameFileIdentity, type FileIdentityStat } from "./file-identity.js"; +import { syncNativeFileBestEffort } from "./native-operations.js"; +import { getNativeBinding, requireNativeBinding, type NativeBinding } from "./native.js"; +import { + directorySyncFailure, + publicationFailure, + rememberCreatedTarget, + type PublishFailureState, + type PublishFileExclusiveCleanup, + type PublishFileExclusiveSyncFailurePolicy, +} from "./publish-file-failure.js"; +import { getFsSafeTestHooks } from "./test-hooks.js"; + +export type { + PublishFileExclusiveCleanup, + PublishFileExclusiveDirectorySyncFailure, + PublishFileExclusiveFailureDetails, + PublishFileExclusiveFailurePhase, + PublishFileExclusiveSyncFailurePolicy, +} from "./publish-file-failure.js"; + +export type PublishFileExclusiveStrategy = + | "link-or-copy" + | "link-required" + | "rename-noreplace"; + +export type PublishFileExclusiveResult = { + method: "hardlink" | "exclusive-copy" | "rename-noreplace"; + identity: Stats; + directorySync: DirectorySyncOutcome; +}; + +const HARDLINK_FALLBACK_CODES = new Set([ + "EPERM", + "EXDEV", + "ENOTSUP", + "EOPNOTSUPP", + "ENOSYS", +]); +const NATIVE_COPY_FALLBACK_CODES = new Set([ + "EINVAL", + "ENOSYS", + "ENOTSUP", + "EOPNOTSUPP", + "EPERM", + "EXDEV", +]); + +export function isHardlinkFallbackError(error: unknown): boolean { + return HARDLINK_FALLBACK_CODES.has((error as NodeJS.ErrnoException | undefined)?.code ?? ""); +} + +function sourceOpenFlags(): number { + return ( + fsSync.constants.O_RDONLY | + (process.platform !== "win32" && typeof fsSync.constants.O_NOFOLLOW === "number" + ? fsSync.constants.O_NOFOLLOW + : 0) + ); +} + +function directoryOpenFlags(): number { + return ( + fsSync.constants.O_RDONLY | + (typeof fsSync.constants.O_DIRECTORY === "number" ? fsSync.constants.O_DIRECTORY : 0) + ); +} + +async function openNativeParent(binding: NativeBinding, filePath: string): Promise<{ + basename: string; + handle: FileHandle; +}> { + const parentPath = path.dirname(filePath); + const handle = await fs.open(parentPath, directoryOpenFlags()); + try { + const pathname = await fs.lstat(parentPath); + const opened = binding.fstatIdentity(handle.fd); + if (pathname.isSymbolicLink() || !sameFileIdentity(pathname, opened)) { + throw new FsSafeError("path-mismatch", "publication parent changed while opening"); + } + return { basename: path.basename(filePath), handle }; + } catch (error) { + await handle.close().catch(() => undefined); + throw error; + } +} + +async function assertPinnedSourceCurrent(params: { + sourcePath: string; + handle: FileHandle; + identity: Stats; +}): Promise { + const opened = await params.handle.stat(); + const current = await fs.lstat(params.sourcePath); + if ( + !opened.isFile() || + current.isSymbolicLink() || + !current.isFile() || + !sameFileIdentity(opened, params.identity) || + !sameFileIdentity(current, opened) + ) { + throw new FsSafeError("path-mismatch", "publication source changed during operation"); + } +} + +async function copyPinnedSource(params: { + source: FileHandle; + targetPath: string; + native?: NativeBinding; + targetNativeParent?: Awaited>; + failure: PublishFailureState; +}): Promise<{ handle: FileHandle; stat: Stats; digest: string; bytes: number }> { + if (params.native && params.targetNativeParent) { + for (const method of ["clone", "copy-file-range"] as const) { + let nativeFd: number | undefined; + try { + if (method === "clone") { + nativeFd = params.native.cloneFileExclusive( + params.source.fd, + params.targetNativeParent.handle.fd, + params.targetNativeParent.basename, + ); + } else { + const copied = await params.native.copyFileRangeExclusive( + params.source.fd, + params.targetNativeParent.handle.fd, + params.targetNativeParent.basename, + ); + if (copied.errorCode) { + throw Object.assign(new Error(copied.errorMessage ?? "native copy failed"), { + code: copied.errorCode, + }); + } + nativeFd = copied.fd; + } + } catch (error) { + if (NATIVE_COPY_FALLBACK_CODES.has((error as NodeJS.ErrnoException).code ?? "")) { + continue; + } + throw error; + } + + let target: FileHandle | undefined; + const createdIdentity = params.native.fstatIdentity(nativeFd); + rememberCreatedTarget(params.failure, createdIdentity, "copy-verify"); + try { + await getFsSafeTestHooks()?.afterPublishTargetCreated?.( + "exclusive-copy", + params.targetPath, + createdIdentity, + ); + const identity = await fs.lstat(params.targetPath); + target = await fs.open(params.targetPath, sourceOpenFlags()); + const opened = await target.stat(); + if ( + identity.isSymbolicLink() || + !identity.isFile() || + !sameFileIdentity(createdIdentity, identity) || + !sameFileIdentity(createdIdentity, opened) + ) { + throw new FsSafeError("path-mismatch", "native publication target changed after copy"); + } + const hashed = await hashFileHandle(target, params.native); + fsSync.closeSync(nativeFd); + nativeFd = undefined; + return { + handle: target, + stat: await target.stat(), + digest: hashed.digest, + bytes: hashed.bytes, + }; + } catch (error) { + await target?.close().catch(() => undefined); + throw error; + } finally { + if (nativeFd !== undefined) fsSync.closeSync(nativeFd); + } + } + } + + const target = await fs.open(params.targetPath, "wx+", 0o600); + const identity = await target.stat(); + rememberCreatedTarget(params.failure, identity, "copy-verify"); + try { + await getFsSafeTestHooks()?.afterPublishTargetCreated?.( + "exclusive-copy", + params.targetPath, + identity, + ); + const hash = createHash("sha256"); + const buffer = Buffer.allocUnsafe(64 * 1024); + let position = 0; + while (true) { + const { bytesRead } = await params.source.read(buffer, 0, buffer.length, position); + if (bytesRead === 0) { + break; + } + hash.update(buffer.subarray(0, bytesRead)); + let written = 0; + while (written < bytesRead) { + const result = await target.write( + buffer, + written, + bytesRead - written, + position + written, + ); + if (result.bytesWritten <= 0) { + throw new FsSafeError("helper-failed", "exclusive publication copy made no progress"); + } + written += result.bytesWritten; + } + position += bytesRead; + } + await target.sync(); + return { handle: target, stat: identity, digest: hash.digest("hex"), bytes: position }; + } catch (error) { + await target.close().catch(() => undefined); + throw error; + } +} + +async function removeCreatedTargetIfUnchanged( + targetPath: string, + identity?: FileIdentityStat, +): Promise { + if (!identity) { + return "unknown"; + } + try { + const current = await fs.lstat(targetPath); + if (!current.isSymbolicLink() && sameFileIdentity(current, identity)) { + await fs.rm(targetPath); + return "removed"; + } + return "preserved"; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ENOENT" ? "removed" : "unknown"; + } +} + +async function syncPublishedParent(params: { + parent: Awaited>; + failure: PublishFailureState; + method: PublishFileExclusiveResult["method"]; + targetPath: string; +}): Promise { + params.failure.phase = "directory-sync"; + try { + await getFsSafeTestHooks()?.beforePublishDirectorySync?.( + params.method, + params.targetPath, + params.failure.targetIdentity!, + ); + // PinnedDirectory.sync() revalidates descriptor/path identity immediately + // before and after fsync; keep that check inside the shared sync boundary. + return await params.parent.sync(); + } catch (error) { + params.failure.directorySync = directorySyncFailure(error); + throw error; + } +} + +export async function publishFileExclusive(params: { + sourcePath: string; + targetPath: string; + expectedSourceIdentity?: FileIdentityStat; + strategy: PublishFileExclusiveStrategy; + onSyncFailure?: PublishFileExclusiveSyncFailurePolicy; + parentReceipt?: DirectoryReceipt; +}): Promise { + const sourcePath = path.resolve(params.sourcePath); + const targetPath = path.resolve(params.targetPath); + const parentPath = path.dirname(targetPath); + if (params.parentReceipt && path.resolve(params.parentReceipt.path) !== parentPath) { + throw new FsSafeError("path-mismatch", "publication parent receipt does not match target parent"); + } + + const sourcePathStat = await fs.lstat(sourcePath); + if (sourcePathStat.isSymbolicLink() || !sourcePathStat.isFile()) { + throw new FsSafeError("not-file", "publication source must be a regular file"); + } + const source = await fs.open(sourcePath, sourceOpenFlags()); + let parent: Awaited> | undefined; + let sourceNativeParent: Awaited> | undefined; + let targetNativeParent: Awaited> | undefined; + const failure: PublishFailureState = { + phase: params.strategy === "rename-noreplace" ? "rename-create" : "hardlink-create", + targetCreated: false, + preserveTarget: false, + }; + try { + parent = await pinDirectory(params.parentReceipt ?? parentPath, { + label: "publication parent", + }); + const sourceIdentity = await source.stat(); + if ( + !sameFileIdentity(sourcePathStat, sourceIdentity) || + (params.expectedSourceIdentity && + !sameFileIdentity(params.expectedSourceIdentity, sourceIdentity)) + ) { + throw new FsSafeError("path-mismatch", "publication source identity did not match"); + } + await parent.assertCurrent(); + await assertPinnedSourceCurrent({ sourcePath, handle: source, identity: sourceIdentity }); + + const native = params.strategy === "rename-noreplace" + ? requireNativeBinding() + : getNativeBinding(); + if (native) { + sourceNativeParent = await openNativeParent(native, sourcePath); + targetNativeParent = await openNativeParent(native, targetPath); + } + + if (params.strategy === "rename-noreplace") { + const binding = requireNativeBinding(); + binding.renameNoReplace( + sourceNativeParent!.handle.fd, + sourceNativeParent!.basename, + targetNativeParent!.handle.fd, + targetNativeParent!.basename, + ); + rememberCreatedTarget(failure, sourceIdentity, "rename-verify"); + // A failed post-rename fence must not delete the only remaining name. + failure.preserveTarget = true; + await getFsSafeTestHooks()?.afterPublishTargetCreated?.( + "rename-noreplace", + targetPath, + sourceIdentity, + ); + const targetIdentity = await fs.lstat(targetPath); + if ( + targetIdentity.isSymbolicLink() || + !targetIdentity.isFile() || + !sameFileIdentity(targetIdentity, sourceIdentity) + ) { + throw new FsSafeError("path-mismatch", "no-replace publication target changed"); + } + try { + await fs.lstat(sourcePath); + throw new FsSafeError("path-mismatch", "no-replace publication source still exists"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + } + syncNativeFileBestEffort(sourceNativeParent!.handle.fd); + return { + method: "rename-noreplace", + identity: targetIdentity, + directorySync: await syncPublishedParent({ + parent, + failure, + method: "rename-noreplace", + targetPath, + }), + }; + } + + try { + if (native) { + native.linkBeneath( + sourceNativeParent!.handle.fd, + sourceNativeParent!.basename, + targetNativeParent!.handle.fd, + targetNativeParent!.basename, + ); + } else { + await fs.link(sourcePath, targetPath); + } + rememberCreatedTarget(failure, sourceIdentity, "hardlink-verify"); + await getFsSafeTestHooks()?.afterPublishTargetCreated?.( + "hardlink", + targetPath, + sourceIdentity, + ); + const targetIdentity = await fs.lstat(targetPath); + if ( + targetIdentity.isSymbolicLink() || + !targetIdentity.isFile() || + !sameFileIdentity(targetIdentity, sourceIdentity) + ) { + throw new FsSafeError("path-mismatch", "hardlink publication target changed"); + } + await assertPinnedSourceCurrent({ sourcePath, handle: source, identity: sourceIdentity }); + return { + method: "hardlink", + identity: targetIdentity, + directorySync: await syncPublishedParent({ + parent, + failure, + method: "hardlink", + targetPath, + }), + }; + } catch (error) { + if ( + failure.targetCreated || + !isHardlinkFallbackError(error) || + params.strategy === "link-required" + ) { + throw error; + } + } + + failure.phase = "copy-create"; + let target: FileHandle | undefined; + let targetIdentity: Stats | undefined; + try { + const copied = await copyPinnedSource({ + source, + targetPath, + native, + targetNativeParent, + failure, + }); + target = copied.handle; + targetIdentity = copied.stat; + const targetPathStat = await fs.lstat(targetPath); + const copiedBack = await hashFileHandle(target, native); + const sourceAfter = await hashFileHandle(source, native); + if ( + targetPathStat.isSymbolicLink() || + !sameFileIdentity(targetPathStat, targetIdentity) || + copiedBack.bytes !== copied.bytes || + copiedBack.digest !== copied.digest || + sourceAfter.bytes !== copied.bytes || + sourceAfter.digest !== copied.digest + ) { + throw new FsSafeError("path-mismatch", "exclusive publication copy failed content fencing"); + } + await assertPinnedSourceCurrent({ sourcePath, handle: source, identity: sourceIdentity }); + const directorySync = await syncPublishedParent({ + parent, + failure, + method: "exclusive-copy", + targetPath, + }); + return { method: "exclusive-copy", identity: targetPathStat, directorySync }; + } catch (error) { + await target?.close().catch(() => undefined); + target = undefined; + throw error; + } finally { + await target?.close().catch(() => undefined); + } + } catch (error) { + if (!failure.targetCreated) throw error; + const preserveSyncFailure = + failure.phase === "directory-sync" && params.onSyncFailure === "preserve"; + const cleanup = failure.preserveTarget || preserveSyncFailure + ? "preserved" + : await removeCreatedTargetIfUnchanged(targetPath, failure.targetIdentity); + throw publicationFailure(error, failure, cleanup); + } finally { + await sourceNativeParent?.handle.close().catch(() => undefined); + await targetNativeParent?.handle.close().catch(() => undefined); + await source.close().catch(() => undefined); + await parent?.close().catch(() => undefined); + } +} diff --git a/src/replace-file.ts b/src/replace-file.ts index 06d0005..a4761c0 100644 --- a/src/replace-file.ts +++ b/src/replace-file.ts @@ -386,12 +386,12 @@ async function replaceFileAtomicUnserialized( const unregisterTempPath = registerTempPathForExit(tempPath); let tempExists = false; let originalError: unknown; - await fsModule.mkdir(dir, { recursive: true, mode: dirMode }); await fsModule.chmod(dir, dirMode).catch(() => undefined); try { tempExists = true; await fsModule.writeFile(tempPath, options.content, { mode, flag: "wx" }); + unregisterTempPath.setIdentity(await fsModule.lstat(tempPath)); if (options.syncTempFile) { await syncTempFile(fsModule, tempPath); } @@ -442,7 +442,6 @@ export function replaceFileAtomicSync( const unregisterTempPath = registerTempPathForExit(tempPath); let tempExists = false; let originalError: unknown; - fsModule.mkdirSync(dir, { recursive: true, mode: dirMode }); try { fsModule.chmodSync(dir, dirMode); @@ -452,6 +451,7 @@ export function replaceFileAtomicSync( try { tempExists = true; fsModule.writeFileSync(tempPath, options.content, { mode, flag: "wx" }); + unregisterTempPath.setIdentity(fsModule.lstatSync(tempPath)); if (options.syncTempFile) { syncTempFileSync(fsModule, tempPath); } diff --git a/src/root-impl.ts b/src/root-impl.ts index 0a57a77..0b90341 100644 --- a/src/root-impl.ts +++ b/src/root-impl.ts @@ -4,12 +4,10 @@ import { constants as fsConstants } from "node:fs"; import type { FileHandle } from "node:fs/promises"; import fs from "node:fs/promises"; import path from "node:path"; -import { pipeline } from "node:stream/promises"; -import { createBoundedReadStream } from "./bounded-read-stream.js"; import { assertAsyncDirectoryGuard, createAsyncDirectoryGuard, createNearestExistingDirectoryGuard } from "./directory-guard.js"; import { FsSafeError } from "./errors.js"; import { syncDirectoryBestEffort } from "./fsync.js"; -import { sameFileIdentity } from "./file-identity.js"; +import { sameFileIdentity, type FileIdentityStat } from "./file-identity.js"; import { mkdirPathComponentsWithGuards } from "./guarded-mkdir.js"; import { withAsyncDirectoryGuards } from "./guarded-mutation.js"; import { @@ -18,13 +16,12 @@ import { type DenyMutationPolicy, } from "./deny-mutations.js"; import { resolveOpenedFileRealPathForHandle } from "./opened-realpath.js"; -import { isPinnedPathHelperSpawnError, runPinnedPathHelper } from "./pinned-path.js"; import { type RenameIdentityPolicy, - runPinnedCopyHelper, + runPinnedWriteHelper, runPinnedWriteWithRenamePolicy, } from "./pinned-write.js"; -import { canFallbackFromPythonError, getFsSafePythonConfig } from "./pinned-python-config.js"; +import { validatePinnedOperationPayload } from "./pinned-operation.js"; import { assertNoPathAliasEscape, PATH_ALIAS_POLICIES } from "./path-policy.js"; import { assertNoNulPathInput, @@ -35,11 +32,6 @@ import { isSymlinkOpenError, } from "./path.js"; import { readOpenedFileSafely, type ReadResult } from "./read-opened-file.js"; -import { - helperReaddir, - helperStat, - runPinnedHelper, -} from "./pinned-helper.js"; import { pathStatFromStats } from "./path-stat.js"; import { resolveRootPath } from "./root-path.js"; import { @@ -58,14 +50,14 @@ import { import { getFsSafeTestHooks } from "./test-hooks.js"; import { stringifyJsonDocument } from "./json-stringify.js"; import type { DirEntry, PathStat } from "./types.js"; -import { registerTempPathForExit } from "./temp-cleanup.js"; +import { walkRoot, type RootWalkEntry, type RootWalkOptions } from "./root-walk.js"; +import { registerTempPathForExit, type TempPathRegistration } from "./temp-cleanup.js"; import { serializePathWrite } from "./write-queue.js"; export type { DenyMutationPolicy } from "./deny-mutations.js"; export type { RenameIdentityPolicy } from "./pinned-write.js"; export { resolveOpenedFileRealPathForHandle } from "./opened-realpath.js"; export type { ReadResult } from "./read-opened-file.js"; - export type OpenResult = { handle: FileHandle; realPath: string; @@ -342,6 +334,7 @@ export interface Root { toRelative: string, options?: RootMoveOptions, ): Promise; + walk(relativePath: string, options: RootWalkOptions): AsyncIterableIterator; } class RootHandle implements Root { @@ -558,14 +551,8 @@ class RootHandle implements Root { async stat(relativePath: string): Promise { assertValidRootRelativePath(relativePath); - try { - return await helperStat(this.rootReal, relativePath); - } catch (error) { - if (canFallbackFromPythonError(error)) { - return await statPathFallback(this.context, relativePath); - } - throw error; - } + validatePinnedOperationPayload({ relativePath }); + return await statPathFallback(this.context, relativePath); } async list(relativePath: string, options?: { withFileTypes?: false }): Promise; @@ -575,16 +562,8 @@ class RootHandle implements Root { options: { withFileTypes?: boolean } = {}, ): Promise { assertValidRootRelativePath(relativePath); - try { - return options.withFileTypes === true - ? await helperReaddir(this.rootReal, relativePath, true) - : await helperReaddir(this.rootReal, relativePath, false); - } catch (error) { - if (canFallbackFromPythonError(error)) { - return await listPathFallback(this.context, relativePath, options.withFileTypes === true); - } - throw error; - } + validatePinnedOperationPayload({ relativePath }); + return await listPathFallback(this.context, relativePath, options.withFileTypes === true); } async move( @@ -594,6 +573,7 @@ class RootHandle implements Root { ): Promise { assertValidRootRelativePath(fromRelative); assertValidRootRelativePath(toRelative); + validatePinnedOperationPayload({ from: fromRelative, to: toRelative }); const denyMutations = mergeDenyMutationPolicies( this.defaults.denyMutations, options.denyMutations, @@ -603,27 +583,18 @@ class RootHandle implements Root { toRelative, denyMutations, }); - try { - await runPinnedHelper("rename", this.rootReal, { - from: fromRelative, - overwrite: options.overwrite ?? false, - to: toRelative, - }); - } catch (error) { - if (canFallbackFromPythonError(error)) { - await movePathFallback(this.context, { - fromRelative, - denyMutations, - overwrite: options.overwrite ?? false, - toRelative, - }); - return; - } - throw error; - } + await movePathFallback(this.context, { + fromRelative, + denyMutations, + overwrite: options.overwrite ?? false, + toRelative, + }); + } + walk(relativePath: string, options: RootWalkOptions): AsyncIterableIterator { + assertValidRootRelativePath(relativePath); + return walkRoot(this, relativePath, options); } } - function readDefaults(defaults: RootDefaults): RootReadParams { return { hardlinks: defaults.hardlinks, @@ -993,26 +964,15 @@ async function removePathInRoot( root: RootContext, params: { relativePath: string; denyMutations?: DenyMutationPolicy }, ): Promise { + validatePinnedOperationPayload({ relativePath: params.relativePath }); const resolved = await resolvePinnedRemovePathInRoot( root, params.relativePath, params.denyMutations, ); - if (process.platform === "win32") { - await removePathFallback(resolved); - return; - } try { - await runPinnedPathHelper({ - operation: "remove", - rootPath: resolved.rootReal, - relativePath: resolved.relativePosix, - }); + await removePathFallback(resolved); } catch (error) { - if (isPinnedPathHelperSpawnError(error)) { - await removePathFallback(resolved); - return; - } throw normalizePinnedPathError(error); } } @@ -1025,22 +985,11 @@ async function mkdirPathInRoot( denyMutations?: DenyMutationPolicy; }, ): Promise { + validatePinnedOperationPayload({ relativePath: params.relativePath }); const resolved = await resolvePinnedPathInRoot(root, params); - if (process.platform === "win32") { - await mkdirPathFallback(resolved); - return; - } try { - await runPinnedPathHelper({ - operation: "mkdirp", - rootPath: resolved.rootReal, - relativePath: resolved.relativePosix, - }); + await mkdirPathFallback(resolved); } catch (error) { - if (isPinnedPathHelperSpawnError(error)) { - await mkdirPathFallback(resolved); - return; - } throw normalizePinnedPathError(error); } } @@ -1137,13 +1086,6 @@ async function copyFileInRoot( } try { - if (process.platform === "win32") { - await serializePathWrite(rootWriteQueueKey(root, params.relativePath), async () => { - await copyFileFallback(root, params, source); - }); - return; - } - const pinned = await resolvePinnedWriteTargetInRoot( root, params.relativePath, @@ -1151,39 +1093,22 @@ async function copyFileInRoot( params.denyMutations, ); await serializePathWrite(pinned.targetPath, async () => { - let identity; + await assertCopySourceCurrent(source); + const identity = await runPinnedWriteHelper({ + rootPath: pinned.rootReal, + relativeParentPath: pinned.relativeParentPath, + basename: pinned.basename, + mkdir: params.mkdir !== false, + mode: pinned.mode, + overwrite: true, + maxBytes: params.maxBytes, + input: { kind: "stream", stream: source.handle.createReadStream() }, + }); try { - if (getFsSafePythonConfig().mode === "off") { - await copyFileFallback(root, params, source); - return; - } - identity = await runPinnedCopyHelper({ - rootPath: pinned.rootReal, - relativeParentPath: pinned.relativeParentPath, - basename: pinned.basename, - mkdir: params.mkdir !== false, - mode: pinned.mode, - overwrite: true, - maxBytes: params.maxBytes, - sourcePath: source.realPath, - sourceIdentity: { dev: source.stat.dev, ino: source.stat.ino }, - }); + await assertCopySourcePathCurrent(source); } catch (error) { - if (canFallbackFromPythonError(error)) { - await copyFileFallback(root, params, source); - return; - } - throw normalizePinnedWriteError(error); - } - try { - await verifyAtomicWriteResult({ - root, - targetPath: pinned.targetPath, - expectedIdentity: identity, - }); - } catch (err) { - emitWriteBoundaryWarning(`post-copy verification failed: ${String(err)}`); - throw err; + await removeCopyTargetIfUnchanged(pinned.targetPath, identity).catch(() => undefined); + throw error; } }); } finally { @@ -1191,6 +1116,35 @@ async function copyFileInRoot( } } +async function assertCopySourceCurrent(source: OpenResult): Promise { + const opened = await source.handle.stat(); + if (!sameFileIdentity(opened, source.stat)) { + throw new FsSafeError("path-mismatch", "copy source descriptor changed"); + } + await assertCopySourcePathCurrent(source); +} + +async function assertCopySourcePathCurrent(source: OpenResult): Promise { + const current = await fs.lstat(source.realPath); + if (current.isSymbolicLink() || !current.isFile() || !sameFileIdentity(current, source.stat)) { + throw new FsSafeError("path-mismatch", "copy source path changed"); + } +} + +async function removeCopyTargetIfUnchanged( + targetPath: string, + identity: FileIdentityStat, +): Promise { + const parentGuard = await createAsyncDirectoryGuard(path.dirname(targetPath)); + const current = await fs.lstat(targetPath); + if (current.isSymbolicLink() || !sameFileIdentity(current, identity)) { + return; + } + await withAsyncDirectoryGuards([parentGuard], async () => { + await fs.rm(targetPath); + }); +} + async function resolvePinnedWriteTargetInRoot( root: RootContext, relativePath: string, @@ -1444,16 +1398,21 @@ async function movePathFallback( relativePath: params.toRelative, policy: PATH_ALIAS_POLICIES.unlinkTarget, }); - try { - await assertNoPathAliasEscape({ - absolutePath: target.resolved, - rootPath: target.rootReal, - boundaryLabel: "root", - }); - } catch (error) { - throw new FsSafeError("path-alias", "path alias escape blocked", { - cause: error instanceof Error ? error : undefined, - }); + const targetStat = await fs.lstat(target.resolved).catch(() => undefined); + const replacesFinalSymlink = + process.platform !== "win32" && params.overwrite && targetStat?.isSymbolicLink() === true; + if (!replacesFinalSymlink) { + try { + await assertNoPathAliasEscape({ + absolutePath: target.resolved, + rootPath: target.rootReal, + boundaryLabel: "root", + }); + } catch (error) { + throw new FsSafeError("path-alias", "path alias escape blocked", { + cause: error instanceof Error ? error : undefined, + }); + } } let sourceStat: Stats; @@ -1542,7 +1501,7 @@ async function writeFileFallback( await target.handle.close().catch(() => {}); const destinationGuard = await createAsyncDirectoryGuard(path.dirname(destinationPath)); let tempPath: string | null = null; - let unregisterTempPath: (() => void) | null = null; + let unregisterTempPath: TempPathRegistration | null = null; try { tempPath = buildAtomicWriteTempPath(destinationPath); unregisterTempPath = registerTempPathForExit(tempPath); @@ -1552,6 +1511,7 @@ async function writeFileFallback( encoding: params.encoding, mode: mode || 0o600, }); + unregisterTempPath.setIdentity(writtenStat); const commitTempPath = tempPath; await withAsyncDirectoryGuards([destinationGuard], async () => { await fs.rename(commitTempPath, destinationPath); @@ -1649,94 +1609,3 @@ async function writeMissingFileFallback( } } } - -async function copyFileFallback( - root: RootContext, - params: { - sourcePath: string; - relativePath: string; - maxBytes?: number; - mkdir?: boolean; - mode?: number; - denyMutations?: DenyMutationPolicy; - sourceHardlinks?: HardlinkPolicy; - }, - source: OpenResult, -): Promise { - let target: WritableOpenResult | null = null; - let sourceClosedByStream = false; - let targetClosedByUs = false; - let tempHandle: FileHandle | null = null; - let tempPath: string | null = null; - let unregisterTempPath: (() => void) | null = null; - let tempClosedByStream = false; - try { - target = await openWritableFileInRoot(root, { - relativePath: params.relativePath, - mkdir: params.mkdir, - mode: params.mode, - denyMutations: params.denyMutations, - truncateExisting: false, - }); - const destinationPath = target.realPath; - const mode = params.mode ?? (target.stat.mode & 0o777); - await target.handle.close().catch(() => {}); - targetClosedByUs = true; - const destinationGuard = await createAsyncDirectoryGuard(path.dirname(destinationPath)); - - tempPath = buildAtomicWriteTempPath(destinationPath); - unregisterTempPath = registerTempPathForExit(tempPath); - tempHandle = await fs.open(tempPath, OPEN_WRITE_CREATE_FLAGS, mode || 0o600); - const sourceStream = createBoundedReadStream(source, params.maxBytes); - const targetStream = tempHandle.createWriteStream(); - sourceStream.once("close", () => { - sourceClosedByStream = true; - }); - targetStream.once("close", () => { - tempClosedByStream = true; - }); - await pipeline(sourceStream, targetStream); - const writtenStat = await fs.stat(tempPath); - if (!tempClosedByStream) { - await tempHandle.close().catch(() => {}); - tempClosedByStream = true; - } - tempHandle = null; - const commitTempPath = tempPath; - await withAsyncDirectoryGuards([destinationGuard], async () => { - await fs.rename(commitTempPath, destinationPath); - }); - tempPath = null; - unregisterTempPath(); - unregisterTempPath = null; - try { - await verifyAtomicWriteResult({ - root, - targetPath: destinationPath, - expectedIdentity: writtenStat, - }); - } catch (err) { - emitWriteBoundaryWarning(`post-copy verification failed: ${String(err)}`); - throw err; - } - } catch (err) { - if (target?.createdForWrite) { - await fs.rm(target.realPath, { force: true }).catch(() => {}); - } - throw err; - } finally { - if (!sourceClosedByStream) { - await source.handle.close().catch(() => {}); - } - if (tempHandle && !tempClosedByStream) { - await tempHandle.close().catch(() => {}); - } - if (tempPath) { - await fs.rm(tempPath, { force: true }).catch(() => {}); - } - unregisterTempPath?.(); - if (target && !targetClosedByUs) { - await target.handle.close().catch(() => {}); - } - } -} diff --git a/src/root-walk.ts b/src/root-walk.ts new file mode 100644 index 0000000..968cbfc --- /dev/null +++ b/src/root-walk.ts @@ -0,0 +1,160 @@ +import path from "node:path"; +import { FsSafeError } from "./errors.js"; +import { resolveRootPath } from "./root-path.js"; +import type { DirEntry } from "./types.js"; + +export type RootWalkSymlinkPolicy = "skip" | "follow-within-root"; +export type RootWalkLimitBehavior = "truncate" | "throw"; +export type RootWalkDirectoryErrorBehavior = "throw" | "skip-and-report"; +export type RootWalkEntryFilterResult = "include" | "skip" | "skip-subtree"; +export type RootWalkDataEntryKind = "file" | "directory" | "other"; +export type RootWalkEntryKind = RootWalkDataEntryKind | "directory-error" | "truncated"; + +export type RootWalkDataEntry = { + relativePath: string; + kind: RootWalkDataEntryKind; + size: number; +}; + +export type RootWalkEntry = + | RootWalkDataEntry + | { relativePath: string; kind: "truncated"; size: 0 } + | { relativePath: string; kind: "directory-error"; size: 0; error: unknown }; + +export type RootWalkEntryFilter = (entry: RootWalkDataEntry) => RootWalkEntryFilterResult; + +export type RootWalkOptions = { + maxDepth?: number; + maxEntries?: number; + symlinkPolicy: RootWalkSymlinkPolicy; + signal?: AbortSignal; + limitBehavior?: RootWalkLimitBehavior; + entryFilter?: RootWalkEntryFilter; + onDirectoryError?: RootWalkDirectoryErrorBehavior; +}; + +type RootWalkCapability = { + rootReal: string; + list(relativePath: string, options: { withFileTypes: true }): Promise; +}; + +function validateBudget(name: string, value: number | undefined): number { + if (value === undefined) return Number.POSITIVE_INFINITY; + if (!Number.isSafeInteger(value) || value < 0) { + throw new RangeError(`${name} must be a non-negative safe integer`); + } + return value; +} + +function entryKind(entry: DirEntry): RootWalkDataEntryKind | "symlink" { + if (entry.isSymbolicLink) return "symlink"; + if (entry.isDirectory) return "directory"; + if (entry.isFile) return "file"; + return "other"; +} + +function limitEntry(relativePath: string): RootWalkEntry { + return { relativePath, kind: "truncated", size: 0 }; +} + +export async function* walkRoot( + root: RootWalkCapability, + relativePath: string, + options: RootWalkOptions, +): AsyncGenerator { + const maxDepth = validateBudget("maxDepth", options.maxDepth); + const maxEntries = validateBudget("maxEntries", options.maxEntries); + const visitedDirectories = new Set(); + let examined = 0; + + const onLimit = (atPath: string): RootWalkEntry => { + if ((options.limitBehavior ?? "truncate") === "throw") { + throw new FsSafeError("too-large", `root walk budget exceeded at ${atPath || "."}`); + } + return limitEntry(atPath); + }; + + async function* visit(directory: string, depth: number): AsyncGenerator { + options.signal?.throwIfAborted(); + let entries: DirEntry[]; + try { + const resolvedDirectory = await resolveRootPath({ + absolutePath: path.resolve(root.rootReal, directory), + rootPath: root.rootReal, + rootCanonicalPath: root.rootReal, + boundaryLabel: "root walk", + }); + if (!resolvedDirectory.exists || resolvedDirectory.kind !== "directory") { + throw new FsSafeError( + "not-file", + `root walk path is not a directory: ${directory || "."}`, + ); + } + if (visitedDirectories.has(resolvedDirectory.canonicalPath)) { + return; + } + visitedDirectories.add(resolvedDirectory.canonicalPath); + + const listingDirectory = path + .relative(root.rootReal, resolvedDirectory.canonicalPath) + .split(path.sep) + .join(path.posix.sep); + entries = await root.list(listingDirectory, { withFileTypes: true }); + } catch (error) { + if ((options.onDirectoryError ?? "throw") === "throw") throw error; + yield { relativePath: directory, kind: "directory-error", size: 0, error }; + return; + } + for (const entry of entries) { + options.signal?.throwIfAborted(); + const child = directory + ? path.posix.join(directory.split(path.sep).join(path.posix.sep), entry.name) + : entry.name; + if (examined >= maxEntries) { + yield onLimit(child); + return; + } + examined += 1; + let kind = entryKind(entry); + let size = entry.size; + if (kind === "symlink") { + if (options.symlinkPolicy === "skip") { + continue; + } + const resolved = await resolveRootPath({ + absolutePath: path.resolve(root.rootReal, child), + rootPath: root.rootReal, + rootCanonicalPath: root.rootReal, + boundaryLabel: "root walk", + }); + if (!resolved.exists) { + continue; + } + kind = resolved.kind === "directory" ? "directory" : resolved.kind === "file" ? "file" : "other"; + size = entry.size; + } + + const walkEntry: RootWalkDataEntry = { relativePath: child, kind, size }; + const filterResult = options.entryFilter?.(walkEntry) ?? "include"; + if (!(["include", "skip", "skip-subtree"] as const).includes(filterResult)) { + throw new TypeError(`invalid root walk entryFilter result: ${String(filterResult)}`); + } + if (filterResult === "include") { + yield walkEntry; + } + if (kind !== "directory") { + continue; + } + if (filterResult === "skip-subtree") { + continue; + } + if (depth >= maxDepth) { + yield onLimit(child); + return; + } + yield* visit(child, depth + 1); + } + } + + yield* visit(relativePath, 0); +} diff --git a/src/root.ts b/src/root.ts index 387dfdd..18a1e5e 100644 --- a/src/root.ts +++ b/src/root.ts @@ -28,3 +28,15 @@ export { type WritableOpenMode, type WritableOpenResult, } from "./root-impl.js"; +export type { + RootWalkDataEntry, + RootWalkDataEntryKind, + RootWalkDirectoryErrorBehavior, + RootWalkEntry, + RootWalkEntryFilter, + RootWalkEntryFilterResult, + RootWalkEntryKind, + RootWalkLimitBehavior, + RootWalkOptions, + RootWalkSymlinkPolicy, +} from "./root-walk.js"; diff --git a/src/secret-file.ts b/src/secret-file.ts index 283ec8e..774f6f6 100644 --- a/src/secret-file.ts +++ b/src/secret-file.ts @@ -314,13 +314,18 @@ async function ensurePrivateDirectory( return { rootGuard, targetReal: await fsp.realpath(resolvedTarget) }; } -export async function writeSecretFileAtomic(params: { +type SecretFileWriteParams = { rootDir: string; filePath: string; content: string | Uint8Array; mode?: number; dirMode?: number; -}): Promise { +}; + +async function materializeSecretFileAtomic( + params: SecretFileWriteParams, + createOnly: boolean, +): Promise { const mode = params.mode ?? PRIVATE_SECRET_FILE_MODE; const dirMode = params.dirMode ?? PRIVATE_SECRET_DIR_MODE; const resolvedRoot = path.resolve(params.rootDir); @@ -340,6 +345,9 @@ export async function writeSecretFileAtomic(params: { try { const stat = await fsp.lstat(finalFilePath); + if (createOnly) { + throw new FsSafeError("secret-exists", `Private secret file ${finalFilePath} already exists.`); + } if (stat.isSymbolicLink()) { throw new Error(`Private secret file ${finalFilePath} must not be a symlink.`); } @@ -360,7 +368,7 @@ export async function writeSecretFileAtomic(params: { basename: fileName, mkdir: false, mode, - overwrite: true, + overwrite: !createOnly, input: { kind: "buffer", data: typeof params.content === "string" ? params.content : Buffer.from(params.content) }, rootIdentity: { dev: parentGuard.stat.dev, ino: parentGuard.stat.ino }, }); @@ -368,3 +376,21 @@ export async function writeSecretFileAtomic(params: { await assertAsyncDirectoryGuard(parentGuard); await enforcePrivateFileIdentityAndMode(finalFilePath, identity, mode); } + +export async function writeSecretFileAtomic(params: SecretFileWriteParams): Promise { + await materializeSecretFileAtomic(params, false); +} + +export async function createSecretFileAtomic(params: SecretFileWriteParams): Promise { + try { + await materializeSecretFileAtomic(params, true); + } catch (error) { + if ( + (error instanceof FsSafeError && error.code === "already-exists") || + (error as NodeJS.ErrnoException).code === "EEXIST" + ) { + throw new FsSafeError("secret-exists", "Private secret file already exists.", { cause: error }); + } + throw error; + } +} diff --git a/src/secret-read-async.ts b/src/secret-read-async.ts new file mode 100644 index 0000000..46907b7 --- /dev/null +++ b/src/secret-read-async.ts @@ -0,0 +1,115 @@ +import fsSync from "node:fs"; +import fs from "node:fs/promises"; +import { readFileHandleBounded } from "./bounded-read.js"; +import { FsSafeError, type FsSafeErrorCode } from "./errors.js"; +import { sameFileIdentity } from "./file-identity.js"; +import { resolveHomeRelativePath } from "./home-dir.js"; +import { + DEFAULT_SECRET_FILE_MAX_BYTES, + type SecretFileReadOptions, +} from "./secret-file.js"; + +type SecretFileReadOutcome = + | { ok: true; secret: string } + | { ok: false; code: FsSafeErrorCode; message: string; error?: unknown }; + +function pathErrorCode(error: unknown): FsSafeErrorCode { + const code = (error as NodeJS.ErrnoException).code; + return code === "ENOENT" || code === "ENOTDIR" ? "not-found" : "invalid-path"; +} + +async function readSecretFileOutcome( + filePath: string, + label: string, + options: SecretFileReadOptions, +): Promise { + const resolvedPath = resolveHomeRelativePath(filePath.trim()); + if (!resolvedPath) { + return { ok: false, code: "invalid-path", message: `${label} file path is empty.` }; + } + const maxBytes = options.maxBytes ?? DEFAULT_SECRET_FILE_MAX_BYTES; + let previewStat; + try { + previewStat = await fs.lstat(resolvedPath); + if (previewStat.isSymbolicLink()) { + if (options.rejectSymlink) { + return { ok: false, code: "symlink", message: `${label} file at ${resolvedPath} must not be a symlink.` }; + } + previewStat = await fs.stat(resolvedPath); + } + } catch (error) { + const normalized = error instanceof Error ? error : new Error(String(error)); + return { + ok: false, + code: pathErrorCode(error), + error: normalized, + message: `Failed to inspect ${label} file at ${resolvedPath}: ${String(normalized)}`, + }; + } + if (!previewStat.isFile()) { + return { ok: false, code: "not-file", message: `${label} file at ${resolvedPath} must be a regular file.` }; + } + if (options.rejectHardlinks !== false && previewStat.nlink > 1) { + return { ok: false, code: "hardlink", message: `${label} file at ${resolvedPath} must not be hardlinked.` }; + } + if (previewStat.size > maxBytes) { + return { ok: false, code: "too-large", message: `${label} file at ${resolvedPath} exceeds ${maxBytes} bytes.` }; + } + + let handle: fs.FileHandle | undefined; + try { + const realPath = await fs.realpath(resolvedPath); + const noFollow = + process.platform !== "win32" && "O_NOFOLLOW" in fsSync.constants + ? fsSync.constants.O_NOFOLLOW + : 0; + handle = await fs.open(realPath, fsSync.constants.O_RDONLY | noFollow); + const openedStat = await handle.stat(); + const pathStat = await fs.lstat(realPath); + if ( + !openedStat.isFile() || + !pathStat.isFile() || + !sameFileIdentity(previewStat, openedStat) || + !sameFileIdentity(pathStat, openedStat) || + (options.rejectHardlinks !== false && openedStat.nlink > 1) + ) { + throw new FsSafeError("path-mismatch", "security validation failed"); + } + const secret = (await readFileHandleBounded(handle, maxBytes)).toString("utf8").trim(); + return secret + ? { ok: true, secret } + : { ok: false, code: "invalid-path", message: `${label} file at ${resolvedPath} is empty.` }; + } catch (error) { + const normalized = error instanceof Error ? error : new Error(String(error)); + return { + ok: false, + code: error instanceof FsSafeError ? error.code : pathErrorCode(error) === "not-found" ? "not-found" : "path-mismatch", + error: normalized, + message: `Failed to read ${label} file at ${resolvedPath}: ${String(normalized)}`, + }; + } finally { + await handle?.close().catch(() => undefined); + } +} + +export async function readSecretFile( + filePath: string, + label: string, + options: SecretFileReadOptions = {}, +): Promise { + const result = await readSecretFileOutcome(filePath, label, options); + if (result.ok) return result.secret; + throw new FsSafeError(result.code, result.message, { cause: result.error }); +} + +export async function tryReadSecretFile( + filePath: string | undefined, + label: string, + options: SecretFileReadOptions = {}, +): Promise { + if (!filePath?.trim()) return undefined; + const result = await readSecretFileOutcome(filePath, label, options); + if (result.ok) return result.secret; + if (result.code === "not-found") return undefined; + throw new FsSafeError(result.code, result.message, { cause: result.error }); +} diff --git a/src/secret.ts b/src/secret.ts index 0e73473..0b90fb7 100644 --- a/src/secret.ts +++ b/src/secret.ts @@ -1,4 +1,5 @@ export { + createSecretFileAtomic, DEFAULT_SECRET_FILE_MAX_BYTES, PRIVATE_SECRET_DIR_MODE, PRIVATE_SECRET_FILE_MODE, @@ -7,3 +8,4 @@ export { writeSecretFileAtomic, type SecretFileReadOptions, } from "./secret-file.js"; +export { readSecretFile, tryReadSecretFile } from "./secret-read-async.js"; diff --git a/src/sibling-temp.ts b/src/sibling-temp.ts index aeba82f..0f826d0 100644 --- a/src/sibling-temp.ts +++ b/src/sibling-temp.ts @@ -72,6 +72,7 @@ export async function writeSiblingTempFile( try { tempExists = true; const result = await options.writeTemp(tempPath); + unregisterTempPath.setIdentity(await fs.lstat(tempPath)); if (options.mode !== undefined) { await fs.chmod(tempPath, options.mode).catch(() => undefined); } diff --git a/src/sidecar-lock-policy.ts b/src/sidecar-lock-policy.ts new file mode 100644 index 0000000..e8fbef4 --- /dev/null +++ b/src/sidecar-lock-policy.ts @@ -0,0 +1,41 @@ +import fs from "node:fs/promises"; +import type { SidecarLockRetryOptions } from "./sidecar-lock-types.js"; + +export function computeSidecarLockDelayMs(retry: SidecarLockRetryOptions, attempt: number): number { + const minTimeout = retry.minTimeout ?? 50; + const maxTimeout = retry.maxTimeout ?? 1000; + const factor = retry.factor ?? 1; + const base = Math.min(maxTimeout, Math.max(minTimeout, minTimeout * factor ** attempt)); + const jitter = retry.randomize ? 1 + Math.random() : 1; + return Math.min(maxTimeout, Math.round(base * jitter)); +} + +export function sidecarLockPayloadIsStale( + payload: unknown, + staleMs: number, + nowMs: number, +): boolean { + const createdAt = + payload && + typeof payload === "object" && + "createdAt" in payload && + typeof payload.createdAt === "string" + ? payload.createdAt + : ""; + const createdAtMs = Date.parse(createdAt); + return Number.isFinite(createdAtMs) && nowMs - createdAtMs > staleMs; +} + +export async function defaultSidecarLockShouldReclaim(params: { + lockPath: string; + payload: unknown; + staleMs: number; + nowMs: number; +}): Promise { + if (sidecarLockPayloadIsStale(params.payload, params.staleMs, params.nowMs)) return true; + try { + return params.nowMs - (await fs.stat(params.lockPath)).mtimeMs > params.staleMs; + } catch { + return true; + } +} diff --git a/src/sidecar-lock-reclaim.ts b/src/sidecar-lock-reclaim.ts index ac96d13..e5e9b3d 100644 --- a/src/sidecar-lock-reclaim.ts +++ b/src/sidecar-lock-reclaim.ts @@ -1,7 +1,13 @@ import { randomBytes } from "node:crypto"; -import type { Stats } from "node:fs"; +import fsSync, { type Stats } from "node:fs"; import fs from "node:fs/promises"; +import path from "node:path"; +import { readFileHandleBounded } from "./bounded-read.js"; +import { FsSafeError } from "./errors.js"; import { sameFileIdentity } from "./file-identity.js"; +import type { Root } from "./root-impl.js"; + +const MAX_LOCK_PAYLOAD_BYTES = 1024 * 1024; const SIDECAR_LOCK_OWNERSHIP_TOKEN_BYTES = 16; const SIDECAR_LOCK_OWNERSHIP_TOKEN_BITS = SIDECAR_LOCK_OWNERSHIP_TOKEN_BYTES * 8; @@ -14,12 +20,12 @@ export type SidecarLockStaleSnapshot = { lockPath: string; normalizedTargetPath: string; raw: string; - payload: Record | null; + payload: unknown; }; export type SidecarLockSnapshot = { raw?: string; - payload: Record | null; + payload: unknown; stat?: Stats; ownershipToken?: string; }; @@ -49,30 +55,109 @@ export function serializeSidecarLockPayload(payload: Record): { }; } +export function relativeSidecarLockPath(lockRoot: Root, lockPath: string): string { + const resolved = path.resolve(lockPath); + const lexicalRelative = path.relative(lockRoot.rootDir, resolved); + const relative = + lexicalRelative !== ".." && + !lexicalRelative.startsWith(`..${path.sep}`) && + !path.isAbsolute(lexicalRelative) + ? lexicalRelative + : path.relative(lockRoot.rootReal, resolved); + if (!relative || relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new FsSafeError("outside-workspace", "sidecar lock path is outside lockRoot"); + } + return relative.split(path.sep).join(path.posix.sep); +} + +export function parseSidecarLockPayload( + raw: string, + parser?: (raw: string) => unknown, +): unknown { + if (parser) { + return parser(raw); + } + try { + const parsed = JSON.parse(raw) as unknown; + return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null; + } catch { + return null; + } +} + export async function readSidecarLockSnapshot( lockPath: string, + options: { lockRoot?: Root; parsePayload?: (raw: string) => unknown } = {}, ): Promise { try { + if (options.lockRoot) { + const opened = await options.lockRoot.open(relativeSidecarLockPath(options.lockRoot, lockPath)); + try { + const raw = (await readFileHandleBounded(opened.handle, MAX_LOCK_PAYLOAD_BYTES)).toString("utf8"); + return { + raw, + payload: parseSidecarLockPayload(raw, options.parsePayload), + stat: opened.stat, + }; + } finally { + await opened.handle.close().catch(() => undefined); + } + } const stat = await fs.lstat(lockPath); const raw = await fs.readFile(lockPath, "utf8"); - try { - const parsed = JSON.parse(raw) as unknown; - const payload = - parsed && typeof parsed === "object" && !Array.isArray(parsed) - ? (parsed as Record) - : null; - return { raw, payload, stat }; - } catch { - return { raw, payload: null, stat }; - } + return { raw, payload: parseSidecarLockPayload(raw, options.parsePayload), stat }; } catch (err) { - if ((err as NodeJS.ErrnoException).code === "ENOENT") { + if ( + (err as NodeJS.ErrnoException).code === "ENOENT" || + (err instanceof FsSafeError && err.code === "not-found") + ) { return null; } throw err; } } +export function readSidecarLockSnapshotSync( + lockPath: string, + parsePayload?: (raw: string) => unknown, +): SidecarLockSnapshot | null { + let fd: number | undefined; + try { + const before = fsSync.lstatSync(lockPath); + if (!before.isFile() || before.isSymbolicLink()) return null; + const noFollow = + process.platform !== "win32" && typeof fsSync.constants.O_NOFOLLOW === "number" + ? fsSync.constants.O_NOFOLLOW + : 0; + fd = fsSync.openSync(lockPath, fsSync.constants.O_RDONLY | noFollow); + const opened = fsSync.fstatSync(fd); + const raw = fsSync.readFileSync(fd, "utf8"); + const after = fsSync.lstatSync(lockPath); + if (!sameFileIdentity(before, opened) || !sameFileIdentity(opened, after)) return null; + return { + raw, + payload: parseSidecarLockPayload(raw, parsePayload), + stat: after, + ownershipToken: readSidecarLockOwnershipToken(raw), + }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } finally { + if (fd !== undefined) fsSync.closeSync(fd); + } +} + +export function removeSidecarLockIfUnchangedSync( + lockPath: string, + observed: SidecarLockSnapshot, +): boolean { + const current = readSidecarLockSnapshotSync(lockPath); + if (!current || !sidecarLockSnapshotMatches(current, observed)) return false; + fsSync.rmSync(lockPath); + return true; +} + export function sidecarLockSnapshotMatches( current: SidecarLockSnapshot, observed: SidecarLockSnapshot, @@ -99,20 +184,26 @@ export function sidecarLockSnapshotMatches( export async function removeSidecarLockIfUnchanged( lockPath: string, observed: SidecarLockSnapshot | null, + options: { lockRoot?: Root; parsePayload?: (raw: string) => unknown } = {}, ): Promise { - const current = await readSidecarLockSnapshot(lockPath); + const current = await readSidecarLockSnapshot(lockPath, options); if (!current || !observed || !sidecarLockSnapshotMatches(current, observed)) { return false; } - await fs.rm(lockPath, { force: true }).catch(() => undefined); + if (options.lockRoot) { + await options.lockRoot.remove(relativeSidecarLockPath(options.lockRoot, lockPath)).catch(() => undefined); + } else { + await fs.rm(lockPath, { force: true }).catch(() => undefined); + } return true; } export async function sidecarLockSnapshotStillPresent( lockPath: string, observed: SidecarLockSnapshot | null, + options: { lockRoot?: Root; parsePayload?: (raw: string) => unknown } = {}, ): Promise { - const current = await readSidecarLockSnapshot(lockPath); + const current = await readSidecarLockSnapshot(lockPath, options); return !!current && !!observed && sidecarLockSnapshotMatches(current, observed); } @@ -157,11 +248,14 @@ export async function removeStaleSidecarLockIfAllowed(params: { normalizedTargetPath: string; snapshot: SidecarLockSnapshot; shouldRemoveStaleLock?: (snapshot: SidecarLockStaleSnapshot) => boolean | Promise; + lockRoot?: Root; + parsePayload?: (raw: string) => unknown; }): Promise<"removed" | "changed" | "not-approved"> { if (!params.shouldRemoveStaleLock || params.snapshot.raw === undefined) { return "not-approved"; } - if (!(await sidecarLockSnapshotStillPresent(params.lockPath, params.snapshot))) { + const ioOptions = { lockRoot: params.lockRoot, parsePayload: params.parsePayload }; + if (!(await sidecarLockSnapshotStillPresent(params.lockPath, params.snapshot, ioOptions))) { return "changed"; } if ( @@ -174,11 +268,15 @@ export async function removeStaleSidecarLockIfAllowed(params: { ) { return "not-approved"; } - if (!(await sidecarLockSnapshotStillPresent(params.lockPath, params.snapshot))) { + if (!(await sidecarLockSnapshotStillPresent(params.lockPath, params.snapshot, ioOptions))) { return "changed"; } try { - await fs.rm(params.lockPath); + if (params.lockRoot) { + await params.lockRoot.remove(relativeSidecarLockPath(params.lockRoot, params.lockPath)); + } else { + await fs.rm(params.lockPath); + } return "removed"; } catch (err) { if ((err as NodeJS.ErrnoException).code === "ENOENT") { diff --git a/src/sidecar-lock-types.ts b/src/sidecar-lock-types.ts new file mode 100644 index 0000000..ca751ec --- /dev/null +++ b/src/sidecar-lock-types.ts @@ -0,0 +1,65 @@ +import type { Root } from "./root-impl.js"; +import type { SidecarLockStaleSnapshot } from "./sidecar-lock-reclaim.js"; + +export type SidecarLockRetryOptions = { + retries?: number; + factor?: number; + minTimeout?: number; + maxTimeout?: number; + randomize?: boolean; +}; + +export type SidecarLockStaleRecovery = "fail-closed" | "remove-if-unchanged"; + +export type SidecarLockCompromisedInfo = { + lockPath: string; + normalizedTargetPath: string; +}; + +export type SidecarLockAcquireOptions> = { + targetPath: string; + lockPath?: string; + staleMs: number; + timeoutMs?: number; + retry?: SidecarLockRetryOptions; + staleRecovery?: SidecarLockStaleRecovery; + allowReentrant?: boolean; + payload: () => TPayload | Promise; + shouldReclaim?: (params: { + lockPath: string; + normalizedTargetPath: string; + payload: unknown; + staleMs: number; + nowMs: number; + heldByThisProcess: boolean; + }) => boolean | Promise; + shouldRemoveStaleLock?: (snapshot: SidecarLockStaleSnapshot) => boolean | Promise; + metadata?: Record; + parsePayload?: (raw: string) => unknown; + lockRoot?: Root; + onCompromised?: (info: SidecarLockCompromisedInfo) => void; + compromiseCheckIntervalMs?: number; +}; + +export type SidecarLockHandle = { + lockPath: string; + normalizedTargetPath: string; + verifyStillHeld: () => Promise; + release: () => Promise; + [Symbol.asyncDispose](): Promise; +}; + +export type SidecarLockHeldEntry = { + normalizedTargetPath: string; + lockPath: string; + acquiredAt: number; + metadata: Record; + forceRelease: () => Promise; +}; + +export type WithSidecarLockOptions> = Omit< + SidecarLockAcquireOptions, + "targetPath" +> & { + managerKey?: string; +}; diff --git a/src/sidecar-lock.ts b/src/sidecar-lock.ts index c62c7b9..9e4699f 100644 --- a/src/sidecar-lock.ts +++ b/src/sidecar-lock.ts @@ -1,10 +1,10 @@ import fsSync from "node:fs"; -import type { FileHandle } from "node:fs/promises"; import fs from "node:fs/promises"; import path from "node:path"; import { sameFileIdentity } from "./file-identity.js"; import { readSidecarLockSnapshot, + relativeSidecarLockPath, releaseSidecarReclaimGuard, removeSidecarLockIfUnchanged, removeStaleSidecarLockIfAllowed, @@ -16,81 +16,49 @@ import { type SidecarLockSnapshot, type SidecarLockStaleSnapshot, } from "./sidecar-lock-reclaim.js"; - +import { FsSafeError } from "./errors.js"; +import { createNativeExclusiveFile, type NativeFileHandle } from "./native-operations.js"; +import type { Root } from "./root-impl.js"; +import type { + SidecarLockAcquireOptions, + SidecarLockHandle, + SidecarLockHeldEntry, + SidecarLockRetryOptions, + WithSidecarLockOptions, +} from "./sidecar-lock-types.js"; +import { + computeSidecarLockDelayMs, + defaultSidecarLockShouldReclaim, +} from "./sidecar-lock-policy.js"; export type { SidecarLockStaleSnapshot } from "./sidecar-lock-reclaim.js"; - -export type SidecarLockRetryOptions = { - retries?: number; - factor?: number; - minTimeout?: number; - maxTimeout?: number; - randomize?: boolean; -}; - -export type SidecarLockStaleRecovery = "fail-closed" | "remove-if-unchanged"; - -export type SidecarLockAcquireOptions> = { - targetPath: string; - lockPath?: string; - staleMs: number; - timeoutMs?: number; - retry?: SidecarLockRetryOptions; - staleRecovery?: SidecarLockStaleRecovery; - allowReentrant?: boolean; - payload: () => TPayload | Promise; - shouldReclaim?: (params: { - lockPath: string; - normalizedTargetPath: string; - payload: Record | null; - staleMs: number; - nowMs: number; - heldByThisProcess: boolean; - }) => boolean | Promise; - shouldRemoveStaleLock?: (snapshot: SidecarLockStaleSnapshot) => boolean | Promise; - metadata?: Record; -}; - -export type SidecarLockHandle = { - lockPath: string; - normalizedTargetPath: string; - release: () => Promise; - [Symbol.asyncDispose](): Promise; -}; - -export type SidecarLockHeldEntry = { - normalizedTargetPath: string; - lockPath: string; - acquiredAt: number; - metadata: Record; - forceRelease: () => Promise; -}; - -export type WithSidecarLockOptions> = Omit< - SidecarLockAcquireOptions, - "targetPath" -> & { - managerKey?: string; -}; - +export type { + SidecarLockAcquireOptions, + SidecarLockCompromisedInfo, + SidecarLockHandle, + SidecarLockHeldEntry, + SidecarLockRetryOptions, + SidecarLockStaleRecovery, + WithSidecarLockOptions, +} from "./sidecar-lock-types.js"; type HeldLock = { count: number; - handle: FileHandle; + handle: NativeFileHandle; lockPath: string; snapshot: SidecarLockSnapshot; acquiredAt: number; metadata: Record; releasePromise?: Promise; + lockRoot?: Root; + parsePayload?: (raw: string) => unknown; + compromiseTimer?: NodeJS.Timeout; }; - type SidecarLockManagerState = { cleanupRegistered: boolean; held: Map; reclaimCleanupRegistered: boolean; reclaimGuards: Set; }; - const GLOBAL_STATE_KEY = Symbol.for("fsSafe.sidecarLockManagers"); - function getGlobalManagers(): Map { const globalWithState = globalThis as typeof globalThis & { [GLOBAL_STATE_KEY]?: Map; @@ -172,34 +140,6 @@ async function resolveNormalizedTargetPath(targetPath: string): Promise } } -function computeDelayMs(retry: SidecarLockRetryOptions, attempt: number): number { - const minTimeout = retry.minTimeout ?? 50; - const maxTimeout = retry.maxTimeout ?? 1000; - const factor = retry.factor ?? 1; - const base = Math.min(maxTimeout, Math.max(minTimeout, minTimeout * factor ** attempt)); - const jitter = retry.randomize ? 1 + Math.random() : 1; - return Math.min(maxTimeout, Math.round(base * jitter)); -} - -async function defaultShouldReclaim(params: { - lockPath: string; - payload: Record | null; - staleMs: number; - nowMs: number; -}): Promise { - const createdAt = typeof params.payload?.createdAt === "string" ? params.payload.createdAt : ""; - const createdAtMs = Date.parse(createdAt); - if (Number.isFinite(createdAtMs) && params.nowMs - createdAtMs > params.staleMs) { - return true; - } - try { - const stat = await fs.stat(params.lockPath); - return params.nowMs - stat.mtimeMs > params.staleMs; - } catch { - return true; - } -} - function releaseAllReclaimGuardsSync(state: SidecarLockManagerState): void { for (const reclaimGuardPath of state.reclaimGuards) { try { @@ -215,7 +155,7 @@ function releaseAllLocksSync(state: SidecarLockManagerState): void { for (const [normalizedTargetPath, held] of state.held) { void held.handle.close().catch(() => undefined); try { - if (snapshotMatchesSync(held.lockPath, held.snapshot)) { + if (!held.lockRoot && snapshotMatchesSync(held.lockPath, held.snapshot)) { fsSync.rmSync(held.lockPath, { force: true }); } } catch { @@ -249,9 +189,16 @@ async function releaseHeldLock( return true; } state.held.delete(normalizedTargetPath); + if (held.compromiseTimer) { + clearInterval(held.compromiseTimer); + held.compromiseTimer = undefined; + } held.releasePromise = (async () => { await held.handle.close().catch(() => undefined); - await removeSidecarLockIfUnchanged(held.lockPath, held.snapshot); + await removeSidecarLockIfUnchanged(held.lockPath, held.snapshot, { + lockRoot: held.lockRoot, + parsePayload: held.parsePayload, + }); })(); try { await held.releasePromise; @@ -288,9 +235,15 @@ export function createSidecarLockManager(key: string) { held.count += 1; const release = () => releaseHeldLock(state, normalizedTargetPath, held).then(() => undefined); + const verifyStillHeld = async () => + await sidecarLockSnapshotStillPresent(held.lockPath, held.snapshot, { + lockRoot: held.lockRoot, + parsePayload: held.parsePayload, + }); return { lockPath, normalizedTargetPath, + verifyStillHeld, release, [Symbol.asyncDispose]: release, }; @@ -320,7 +273,7 @@ export function createSidecarLockManager(key: string) { options.timeoutMs === undefined || options.timeoutMs === Number.POSITIVE_INFINITY ? Number.POSITIVE_INFINITY : Math.max(0, options.timeoutMs - elapsed); - const delay = Math.min(computeDelayMs(retry, attempt), remaining); + const delay = Math.min(computeSidecarLockDelayMs(retry, attempt), remaining); attempt += 1; await new Promise((resolve) => setTimeout(resolve, delay)); }; @@ -330,12 +283,25 @@ export function createSidecarLockManager(key: string) { await waitForRetry(); continue; } - let handle: FileHandle | null = null; + let handle: NativeFileHandle | null = null; try { - handle = await fs.open(lockPath, "wx"); const payload = await options.payload(); const { raw, ownershipToken } = serializeSidecarLockPayload(payload); - await handle.writeFile(raw, "utf8"); + if (options.lockRoot) { + const relativeLockPath = relativeSidecarLockPath(options.lockRoot, lockPath); + try { + await options.lockRoot.create(relativeLockPath, raw, { mkdir: true, mode: 0o600 }); + } catch (error) { + if (error instanceof FsSafeError && error.code === "already-exists") { + throw Object.assign(new Error("sidecar lock exists"), { code: "EEXIST" }); + } + throw error; + } + handle = (await options.lockRoot.open(relativeLockPath)).handle; + } else { + handle = (await createNativeExclusiveFile(lockPath, 0o600)) ?? await fs.open(lockPath, "wx"); + await handle.writeFile(raw, "utf8"); + } const snapshot = { raw, payload, stat: await handle.stat(), ownershipToken }; const createdHeld: HeldLock = { count: 1, @@ -344,6 +310,8 @@ export function createSidecarLockManager(key: string) { snapshot, acquiredAt: Date.now(), metadata: options.metadata ?? {}, + lockRoot: options.lockRoot, + parsePayload: options.parsePayload, }; state.held.set(normalizedTargetPath, createdHeld); if (ownsReclaimGuard) { @@ -357,9 +325,28 @@ export function createSidecarLockManager(key: string) { } const release = () => releaseHeldLock(state, normalizedTargetPath, createdHeld).then(() => undefined); + const verifyStillHeld = async () => + await sidecarLockSnapshotStillPresent(lockPath, snapshot, { + lockRoot: options.lockRoot, + parsePayload: options.parsePayload, + }); + const interval = options.compromiseCheckIntervalMs; + if (options.onCompromised && interval !== undefined && interval > 0) { + createdHeld.compromiseTimer = setInterval(() => { + void verifyStillHeld().then((stillHeld) => { + if (!stillHeld && createdHeld.compromiseTimer) { + clearInterval(createdHeld.compromiseTimer); + createdHeld.compromiseTimer = undefined; + options.onCompromised?.({ lockPath, normalizedTargetPath }); + } + }); + }, interval); + createdHeld.compromiseTimer.unref(); + } return { lockPath, normalizedTargetPath, + verifyStillHeld, release, [Symbol.asyncDispose]: release, }; @@ -377,11 +364,16 @@ export function createSidecarLockManager(key: string) { } // If payload serialization/write fails, the file may be empty or // partial JSON, so remove while our exclusive handle is still open. - await fs.rm(lockPath, { force: true }).catch(() => undefined); + if (!options.lockRoot) { + await fs.rm(lockPath, { force: true }).catch(() => undefined); + } await handle.close().catch(() => undefined); // Windows can refuse removing an open file; retry after close but // only if the path still points at the file identity we created. - await removeSidecarLockIfUnchanged(lockPath, failedSnapshot); + await removeSidecarLockIfUnchanged(lockPath, failedSnapshot, { + lockRoot: options.lockRoot, + parsePayload: options.parsePayload, + }); } if ((err as { code?: unknown }).code !== "EEXIST") { throw err; @@ -392,11 +384,14 @@ export function createSidecarLockManager(key: string) { continue; } const nowMs = Date.now(); - const snapshot = await readSidecarLockSnapshot(lockPath); + const snapshot = await readSidecarLockSnapshot(lockPath, { + lockRoot: options.lockRoot, + parsePayload: options.parsePayload, + }); if (!snapshot) { continue; } - const shouldReclaim = options.shouldReclaim ?? defaultShouldReclaim; + const shouldReclaim = options.shouldReclaim ?? defaultSidecarLockShouldReclaim; if ( await shouldReclaim({ lockPath, @@ -407,7 +402,12 @@ export function createSidecarLockManager(key: string) { heldByThisProcess: state.held.has(normalizedTargetPath), }) ) { - if (!(await sidecarLockSnapshotStillPresent(lockPath, snapshot))) { + if ( + !(await sidecarLockSnapshotStillPresent(lockPath, snapshot, { + lockRoot: options.lockRoot, + parsePayload: options.parsePayload, + })) + ) { continue; } const staleRecovery = options.staleRecovery ?? "fail-closed"; @@ -422,6 +422,8 @@ export function createSidecarLockManager(key: string) { normalizedTargetPath, snapshot, shouldRemoveStaleLock: options.shouldRemoveStaleLock, + lockRoot: options.lockRoot, + parsePayload: options.parsePayload, }); if (removal === "removed" || removal === "changed") { continue; diff --git a/src/temp-cleanup.ts b/src/temp-cleanup.ts index 72cea42..e0dcd7f 100644 --- a/src/temp-cleanup.ts +++ b/src/temp-cleanup.ts @@ -1,17 +1,38 @@ import fsSync from "node:fs"; +import { sameFileIdentity, type FileIdentityStat } from "./file-identity.js"; + +export type TempPathIdentityReceipt = FileIdentityStat; + +export type TempPathRegistration = (() => void) & { + setIdentity(identity: FileIdentityStat): void; +}; type TempCleanupEntry = { path: string; recursive: boolean; + identity?: FileIdentityStat; }; const tempCleanupEntries = new Map(); let cleanupRegistered = false; +function pathStillMatchesReceipt(entry: TempCleanupEntry): boolean { + if (!entry.identity) { + return false; + } + try { + return sameFileIdentity(fsSync.lstatSync(entry.path), entry.identity); + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ENOENT"; + } +} + function cleanupRegisteredTempPathsSync(): void { for (const entry of tempCleanupEntries.values()) { try { - fsSync.rmSync(entry.path, { force: true, recursive: entry.recursive }); + if (pathStillMatchesReceipt(entry)) { + fsSync.rmSync(entry.path, { force: true, recursive: entry.recursive }); + } } catch { // Process-exit cleanup is best-effort. } @@ -21,19 +42,32 @@ function cleanupRegisteredTempPathsSync(): void { export function registerTempPathForExit( tempPath: string, - options?: { recursive?: boolean }, -): () => void { + options?: { recursive?: boolean; identity?: FileIdentityStat }, +): TempPathRegistration { if (!cleanupRegistered) { cleanupRegistered = true; process.once("exit", cleanupRegisteredTempPathsSync); } - tempCleanupEntries.set(tempPath, { + const entry: TempCleanupEntry = { path: tempPath, recursive: options?.recursive === true, - }); - return () => { + identity: options?.identity, + }; + if (!entry.identity) { + try { + entry.identity = fsSync.lstatSync(tempPath); + } catch { + // Callers that register before creation set the identity after opening. + } + } + tempCleanupEntries.set(tempPath, entry); + const unregister = (() => { tempCleanupEntries.delete(tempPath); + }) as TempPathRegistration; + unregister.setIdentity = (identity) => { + entry.identity = identity; }; + return unregister; } export function __cleanupRegisteredTempPathsForTest(): void { @@ -46,7 +80,9 @@ export function __cleanupRegisteredTempPathForTest(tempPath: string): void { return; } try { - fsSync.rmSync(entry.path, { force: true, recursive: entry.recursive }); + if (pathStillMatchesReceipt(entry)) { + fsSync.rmSync(entry.path, { force: true, recursive: entry.recursive }); + } } finally { tempCleanupEntries.delete(tempPath); } diff --git a/src/temp.ts b/src/temp.ts index f95c607..b595a43 100644 --- a/src/temp.ts +++ b/src/temp.ts @@ -2,9 +2,11 @@ export { tempWorkspace, type TempWorkspace, type TempWorkspaceOptions, + type TempWorkspaceCleanupResult, tempWorkspaceSync, type TempWorkspaceSync, withTempWorkspace, withTempWorkspaceSync, } from "./private-temp-workspace.js"; +export type { TempPathIdentityReceipt } from "./temp-cleanup.js"; export { resolveSecureTempRoot, type ResolveSecureTempRootOptions } from "./secure-temp-dir.js"; diff --git a/src/test-hooks.ts b/src/test-hooks.ts index b25aba8..de56fdf 100644 --- a/src/test-hooks.ts +++ b/src/test-hooks.ts @@ -1,4 +1,5 @@ import type { FileHandle } from "node:fs/promises"; +import type { FileIdentityStat } from "./file-identity.js"; export type FsSafeTestHooks = { afterPreOpenLstat?: (filePath: string) => Promise | void; @@ -17,6 +18,16 @@ export type FsSafeTestHooks = { afterPinnedWriteFallbackRename?: (targetPath: string) => Promise | void; beforeSiblingTempWrite?: (tempPath: string) => Promise | void; beforeTrashMove?: (targetPath: string, destPath: string) => void; + afterPublishTargetCreated?: ( + method: "hardlink" | "exclusive-copy" | "rename-noreplace", + targetPath: string, + identity: FileIdentityStat, + ) => Promise | void; + beforePublishDirectorySync?: ( + method: "hardlink" | "exclusive-copy" | "rename-noreplace", + targetPath: string, + identity: FileIdentityStat, + ) => Promise | void; }; let fsSafeTestHooks: FsSafeTestHooks | undefined; diff --git a/src/windows-permissions-native.ts b/src/windows-permissions-native.ts new file mode 100644 index 0000000..e6a9dfd --- /dev/null +++ b/src/windows-permissions-native.ts @@ -0,0 +1,37 @@ +import { getNativeBinding } from "./native.js"; +import type { PermissionCheck, SafeStatResult } from "./permissions.js"; + +export function inspectWindowsPermissionsNative(params: { + targetPath: string; + stat: SafeStatResult; + effectiveIsDir: boolean; + effectiveMode: number | null; + bits: number | null; +}): PermissionCheck | undefined { + const native = getNativeBinding(); + if (!native) return undefined; + try { + const facts = native.readOwnerAndDacl(params.targetPath); + if (facts.fallbackRequired) return undefined; + return { + ok: true, + isSymlink: params.stat.isSymlink, + isDir: params.effectiveIsDir, + mode: params.effectiveMode, + bits: params.bits, + source: "windows-acl", + worldWritable: facts.worldWritable, + groupWritable: facts.groupWritable, + worldReadable: facts.worldReadable, + groupReadable: facts.groupReadable, + ownerSid: facts.ownerSid, + ownerTrusted: facts.ownerClass !== "foreign", + aclSummary: + `native owner=${facts.ownerClass} world=` + + `${facts.worldReadable ? "r" : "-"}${facts.worldWritable ? "w" : "-"} ` + + `group=${facts.groupReadable ? "r" : "-"}${facts.groupWritable ? "w" : "-"}`, + }; + } catch { + return undefined; + } +} diff --git a/test/adversarial-boundary-payloads.test.ts b/test/adversarial-boundary-payloads.test.ts index b1bb1e4..ae1cd83 100644 --- a/test/adversarial-boundary-payloads.test.ts +++ b/test/adversarial-boundary-payloads.test.ts @@ -104,7 +104,7 @@ describe("adversarial boundary payloads", () => { await attemptAll(safeRoot, payload); await expectOutsideUntouched(layout); } - }, 15_000); + }, 30_000); it("rejects chained symlink parent escapes across read and write surfaces", async () => { const layout = await makeTempLayout("fs-safe-symlink-chain"); @@ -140,5 +140,5 @@ describe("adversarial boundary payloads", () => { await expectOutsideUntouched(layout); await fsp.writeFile(source, "source"); } - }, 15_000); + }, 30_000); }); diff --git a/test/archive-policy.test.ts b/test/archive-policy.test.ts new file mode 100644 index 0000000..bcdc1bd --- /dev/null +++ b/test/archive-policy.test.ts @@ -0,0 +1,127 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import JSZip from "jszip"; +import * as tar from "tar"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { extractArchive, readArchiveEntry } from "../src/archive.js"; +import { + __resetFsSafeNativeConfigForTest, + configureFsSafeNative, +} from "../src/native-config.js"; + +const tempDirs: string[] = []; + +beforeEach(() => { + configureFsSafeNative({ mode: "off" }); +}); + +async function tempRoot(prefix: string): Promise { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); + tempDirs.push(directory); + return directory; +} + +afterEach(async () => { + __resetFsSafeNativeConfigForTest(); + await Promise.all(tempDirs.splice(0).map((directory) => fs.rm(directory, { recursive: true, force: true }))); +}); + +describe("archive entry policy", () => { + it.runIf(process.platform !== "win32")("clamps ZIP modes by default and preserves safe permission bits on request", async () => { + const root = await tempRoot("fs-safe-archive-policy-"); + const archivePath = path.join(root, "package.zip"); + const zip = new JSZip(); + zip.file("plain.txt", "plain", { unixPermissions: 0o10670 }); + zip.file("tool", "exec", { unixPermissions: 0o10711 }); + await fs.writeFile(archivePath, await zip.generateAsync({ type: "nodebuffer", platform: "UNIX" })); + + const clamped = path.join(root, "clamped"); + await fs.mkdir(clamped); + await extractArchive({ archivePath, destDir: clamped, timeoutMs: 10_000 }); + expect((await fs.stat(path.join(clamped, "plain.txt"))).mode & 0o7777).toBe(0o644); + expect((await fs.stat(path.join(clamped, "tool"))).mode & 0o7777).toBe(0o755); + + const preserved = path.join(root, "preserved"); + await fs.mkdir(preserved); + await extractArchive({ + archivePath, + destDir: preserved, + timeoutMs: 10_000, + entryModes: "preserve", + }); + expect((await fs.stat(path.join(preserved, "plain.txt"))).mode & 0o7777).toBe(0o670); + expect((await fs.stat(path.join(preserved, "tool"))).mode & 0o7777).toBe(0o711); + }); + + it("rejects filtered archives by default and can explicitly skip entries", async () => { + const root = await tempRoot("fs-safe-archive-filter-"); + const archivePath = path.join(root, "package.zip"); + const zip = new JSZip(); + zip.file("keep.txt", "keep"); + zip.file("skip.txt", "skip"); + await fs.writeFile(archivePath, await zip.generateAsync({ type: "nodebuffer" })); + const rejected = path.join(root, "rejected"); + await fs.mkdir(rejected); + const filter = ({ path: entryPath }: { path: string }) => + entryPath === "skip.txt" ? ("skip" as const) : ("extract" as const); + + await expect( + extractArchive({ archivePath, destDir: rejected, timeoutMs: 10_000, entryFilter: filter }), + ).rejects.toThrow("archive entry rejected by filter"); + await expect(fs.readdir(rejected)).resolves.toEqual([]); + + const skipped = path.join(root, "skipped"); + await fs.mkdir(skipped); + await extractArchive({ + archivePath, + destDir: skipped, + timeoutMs: 10_000, + entryFilter: filter, + onFiltered: "skip-entry", + }); + await expect(fs.readFile(path.join(skipped, "keep.txt"), "utf8")).resolves.toBe("keep"); + await expect(fs.access(path.join(skipped, "skip.txt"))).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it.runIf(process.platform !== "win32")("applies TAR mode and filtering policy in staging", async () => { + const root = await tempRoot("fs-safe-tar-policy-"); + const input = path.join(root, "input"); + await fs.mkdir(input); + await fs.writeFile(path.join(input, "tool"), "tool"); + await fs.writeFile(path.join(input, "skip"), "skip"); + await fs.chmod(path.join(input, "tool"), 0o710); + const archivePath = path.join(root, "package.tar"); + await tar.c({ cwd: input, file: archivePath }, ["tool", "skip"]); + const destination = path.join(root, "destination"); + await fs.mkdir(destination); + await extractArchive({ + archivePath, + destDir: destination, + timeoutMs: 10_000, + entryFilter: (entry) => (entry.path === "skip" ? "skip" : "extract"), + onFiltered: "skip-entry", + }); + expect((await fs.stat(path.join(destination, "tool"))).mode & 0o7777).toBe(0o755); + await expect(fs.access(path.join(destination, "skip"))).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("reads bounded ZIP and TAR entries without materializing a destination", async () => { + const root = await tempRoot("fs-safe-archive-read-"); + const zipPath = path.join(root, "package.zip"); + const zip = new JSZip(); + zip.file("nested/value.txt", "value"); + await fs.writeFile(zipPath, await zip.generateAsync({ type: "nodebuffer" })); + await expect(readArchiveEntry(zipPath, "nested/value.txt", { maxBytes: 5 })).resolves.toEqual(Buffer.from("value")); + await expect(readArchiveEntry(zipPath, "nested/value.txt", { maxBytes: 4 })).rejects.toMatchObject({ + code: "archive-entry-extracted-size-exceeds-limit", + }); + + const input = path.join(root, "input"); + await fs.mkdir(input); + await fs.writeFile(path.join(input, "value.txt"), "tar-value"); + const tarPath = path.join(root, "package.tar"); + await tar.c({ cwd: input, file: tarPath }, ["value.txt"]); + await expect(readArchiveEntry(tarPath, "value.txt", { maxBytes: 9 })).resolves.toEqual(Buffer.from("tar-value")); + }); +}); diff --git a/test/archive.test.ts b/test/archive.test.ts index bc4f3b6..2ac19c2 100644 --- a/test/archive.test.ts +++ b/test/archive.test.ts @@ -4,7 +4,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import JSZip from "jszip"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { ARCHIVE_LIMIT_ERROR_CODE, type ArchiveSecurityError, @@ -12,6 +12,7 @@ import { resolvePackedRootDir, } from "../src/archive.js"; import { withExtractionDeadline } from "../src/archive-deadline.js"; +import { __resetFsSafeNativeConfigForTest, configureFsSafeNative } from "../src/native-config.js"; import { __setFsSafeTestHooksForTest } from "../src/test-hooks.js"; import { buildRandomTempFilePath, @@ -22,6 +23,10 @@ import { const tempDirs: string[] = []; +beforeEach(() => { + configureFsSafeNative({ mode: "off" }); +}); + async function tempRoot(prefix: string): Promise { const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); tempDirs.push(dir); @@ -71,6 +76,7 @@ async function withRealpathSymlinkRebindRace(params: { } afterEach(async () => { + __resetFsSafeNativeConfigForTest(); __setFsSafeTestHooksForTest(undefined); await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { force: true, recursive: true }))); }); @@ -108,6 +114,38 @@ describe("archive extraction", () => { await expect(fs.readFile(path.join(packageDir, "my file.txt"), "utf8")).resolves.toBe("space"); }); + it("supports buffer-only ZIP entries while stripping the archive root", async () => { + const root = await tempRoot("fs-safe-archive-buffer-only-"); + const archivePath = path.join(root, "pkg.zip"); + const destDir = path.join(root, "dest"); + await fs.mkdir(destDir); + const zip = new JSZip(); + zip.file("package/hello.txt", "buffer-only"); + const bytes = await zip.generateAsync({ type: "nodebuffer" }); + await fs.writeFile(archivePath, bytes); + + const loaded = await JSZip.loadAsync(bytes); + const entry = loaded.file("package/hello.txt"); + const prototype = Object.getPrototypeOf(entry) as object; + const descriptor = Object.getOwnPropertyDescriptor(prototype, "nodeStream"); + expect(descriptor).toBeDefined(); + Object.defineProperty(prototype, "nodeStream", { ...descriptor, value: undefined }); + try { + await extractArchive({ + archivePath, + destDir, + kind: "zip", + stripComponents: 1, + timeoutMs: 15_000, + }); + } finally { + Object.defineProperty(prototype, "nodeStream", descriptor!); + } + await expect(fs.readFile(path.join(destDir, "hello.txt"), "utf8")).resolves.toBe( + "buffer-only", + ); + }); + it("copies every byte when staging archive input after short writes", async () => { const root = await tempRoot("fs-safe-archive-short-write-"); const archivePath = path.join(root, "pkg.zip"); diff --git a/test/clawpatch-regression.test.ts b/test/clawpatch-regression.test.ts index 1e61e57..408846b 100644 --- a/test/clawpatch-regression.test.ts +++ b/test/clawpatch-regression.test.ts @@ -13,9 +13,8 @@ import { writeJsonDurableQueueEntry, } from "../src/json-durable-queue.js"; import { resolveSafeRelativePath } from "../src/path.js"; -import { __resetPinnedPythonWorkerForTest } from "../src/pinned-python.js"; import { summarizeWindowsAcl } from "../src/permissions.js"; -import { configureFsSafePython, root as openRoot } from "../src/index.js"; +import { configureFsSafeNative, root as openRoot } from "../src/index.js"; import { resolveExistingPathsWithinRoot, resolvePathWithinRoot } from "../src/root-paths.js"; import { readSecureFile } from "../src/secure-file.js"; import { withTimeout } from "../src/timing.js"; @@ -30,8 +29,7 @@ async function tempRoot(prefix: string): Promise { afterEach(async () => { vi.restoreAllMocks(); - __resetPinnedPythonWorkerForTest(); - configureFsSafePython({ mode: "auto", pythonPath: undefined }); + configureFsSafeNative({ mode: "auto" }); await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); }); @@ -375,7 +373,7 @@ describe("clawpatch regression coverage", () => { }); it.runIf(process.platform !== "win32")("uses atomic no-clobber writes for root create", async () => { - configureFsSafePython({ mode: "require" }); + configureFsSafeNative({ mode: "auto" }); const rootDir = await tempRoot("fs-safe-root-create-noclobber-"); const scoped = await openRoot(rootDir); await scoped.create("created.txt", "first"); @@ -387,7 +385,7 @@ describe("clawpatch regression coverage", () => { }); it.runIf(process.platform !== "win32")("keeps already-exists for no-clobber creates in read-only parents", async () => { - configureFsSafePython({ mode: "require" }); + configureFsSafeNative({ mode: "auto" }); const rootDir = await tempRoot("fs-safe-root-create-existing-readonly-"); const parent = path.join(rootDir, "readonly"); await fs.mkdir(parent); @@ -405,7 +403,7 @@ describe("clawpatch regression coverage", () => { }); it.runIf(process.platform !== "win32")("rolls back no-clobber move links when source unlink fails", async () => { - configureFsSafePython({ mode: "require" }); + configureFsSafeNative({ mode: "auto" }); const rootDir = await tempRoot("fs-safe-root-move-link-rollback-"); const srcDir = path.join(rootDir, "src"); const dstDir = path.join(rootDir, "dst"); @@ -427,8 +425,8 @@ describe("clawpatch regression coverage", () => { }); it.runIf(process.platform !== "win32")("rejects no-clobber directory moves instead of racing rename", async () => { - for (const mode of ["require", "off"] as const) { - configureFsSafePython({ mode }); + for (const mode of ["auto", "off"] as const) { + configureFsSafeNative({ mode }); const rootDir = await tempRoot(`fs-safe-root-move-dir-noclobber-${mode}-`); await fs.mkdir(path.join(rootDir, "from")); const scoped = await openRoot(rootDir); @@ -440,35 +438,19 @@ describe("clawpatch regression coverage", () => { }); - it.runIf(process.platform !== "win32")("cleans no-clobber copy fallback destinations after helper copy failure", async () => { - __resetPinnedPythonWorkerForTest(); + it("cleans no-clobber fallback destinations after a write failure", async () => { const rootDir = await tempRoot("fs-safe-root-create-copy-fail-"); - const wrapperPath = path.join(rootDir, "python-wrapper.mjs"); - await fs.writeFile( - wrapperPath, - `#!/usr/bin/env node -import { spawn } from "node:child_process"; - -const args = process.argv.slice(2); -const sourceIndex = args.indexOf("-c") + 1; -if (sourceIndex <= 0 || sourceIndex >= args.length) { - process.exit(64); -} -const source = args[sourceIndex] - .replaceAll("os.link(temp_name, basename, src_dir_fd=parent_fd, dst_dir_fd=parent_fd, follow_symlinks=False)", "raise OSError(errno.EPERM, 'link unsupported')") - .replaceAll("os.link(name, new_name, src_dir_fd=source_fd, dst_dir_fd=target_fd, follow_symlinks=False)", "raise OSError(errno.EPERM, 'link unsupported')") - .replaceAll(" os.fsync(dest_fd)", " raise OSError(errno.EIO, 'test dest fsync failure')"); -args[sourceIndex] = source; -const child = spawn("python3", args, { stdio: "inherit" }); -child.on("exit", (code, signal) => { - if (signal) process.kill(process.pid, signal); - process.exit(code ?? 1); -}); -`, - { mode: 0o700 }, - ); - await fs.chmod(wrapperPath, 0o700); - configureFsSafePython({ mode: "require", pythonPath: wrapperPath }); + configureFsSafeNative({ mode: "off" }); + const realOpen = fs.open.bind(fs); + vi.spyOn(fs, "open").mockImplementation(async (...args) => { + const handle = await realOpen(...args); + if (path.basename(String(args[0])) === "created.txt") { + vi.spyOn(handle, "writeFile").mockRejectedValueOnce( + Object.assign(new Error("test write failure"), { code: "EIO" }), + ); + } + return handle; + }); const scoped = await openRoot(rootDir); await expect(scoped.create("created.txt", "payload")).rejects.toBeTruthy(); diff --git a/test/deepsec-regression.test.ts b/test/deepsec-regression.test.ts index 2aa6614..01122ea 100644 --- a/test/deepsec-regression.test.ts +++ b/test/deepsec-regression.test.ts @@ -3,10 +3,9 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { fileStore } from "../src/file-store.js"; -import { configureFsSafePython, root as openRoot } from "../src/index.js"; +import { configureFsSafeNative, root as openRoot } from "../src/index.js"; import { loadPendingJsonDurableQueueEntries } from "../src/json-durable-queue.js"; import { readLocalFileFromRoots, resolveLocalPathFromRootsSync } from "../src/local-roots.js"; -import { __resetPinnedPythonWorkerForTest, runPinnedPythonOperation } from "../src/pinned-python.js"; import { replaceFileAtomic } from "../src/replace-file.js"; import { resolveRootPath } from "../src/root-path.js"; import { assertNoSymlinkParents } from "../src/symlink-parents.js"; @@ -24,8 +23,7 @@ async function tempRoot(prefix: string): Promise { afterEach(async () => { vi.restoreAllMocks(); - __resetPinnedPythonWorkerForTest(); - configureFsSafePython({ mode: "auto", pythonPath: undefined }); + configureFsSafeNative({ mode: "auto" }); await Promise.all(tempDirs.splice(0).map((dir) => fsp.rm(dir, { recursive: true, force: true }))); }); @@ -211,7 +209,7 @@ describe("deepsec regressions", () => { }); it.runIf(process.platform !== "win32")("rejects fallback writes when a missing parent is raced to a symlink", async () => { - configureFsSafePython({ mode: "off" }); + configureFsSafeNative({ mode: "off" }); const base = await tempRoot("fs-safe-fallback-mkdir-symlink-"); const rootDir = path.join(base, "root"); const outside = path.join(base, "outside"); @@ -258,9 +256,9 @@ describe("deepsec regressions", () => { await expect(fsp.lstat(path.join(outside, "nested"))).rejects.toMatchObject({ code: "ENOENT" }); }); - it.runIf(process.platform !== "win32")("uses best-effort private secret writes without the pinned helper", async () => { - configureFsSafePython({ mode: "off" }); - const base = await tempRoot("fs-safe-secret-helper-required-"); + it.runIf(process.platform !== "win32")("uses guarded private secret writes with native mode off", async () => { + configureFsSafeNative({ mode: "off" }); + const base = await tempRoot("fs-safe-secret-native-off-"); const rootDir = path.join(base, "root"); await fsp.mkdir(rootDir); const secretPath = path.join(rootDir, "secret.txt"); @@ -356,16 +354,4 @@ describe("deepsec regressions", () => { expect((await fsp.stat(outsideFile)).mode & 0o777).toBe(0o600); }); - it.runIf(process.platform !== "win32")("rejects pinned helper operations after root swaps", async () => { - const rootDir = await tempRoot("fs-safe-helper-root-identity-"); - const stat = await fsp.lstat(rootDir); - - await expect( - runPinnedPythonOperation({ - operation: "stat", - rootPath: rootDir, - payload: { relativePath: "", rootDev: stat.dev + 1, rootIno: stat.ino }, - }), - ).rejects.toMatchObject({ code: "path-mismatch" }); - }); }); diff --git a/test/durability-exports.test.ts b/test/durability-exports.test.ts index 78e1732..1c7b6a9 100644 --- a/test/durability-exports.test.ts +++ b/test/durability-exports.test.ts @@ -5,7 +5,10 @@ describe("durability exports", () => { it("exposes the public directory durability surface", () => { expect(Object.keys(durability).toSorted()).toEqual([ "ensureDurableDirectory", + "isHardlinkFallbackError", "pinDirectory", + "publishFileExclusive", + "sha256File", "syncDirectory", "syncDirectoryBestEffort", "syncDirectoryBestEffortSync", diff --git a/test/file-hash.test.ts b/test/file-hash.test.ts new file mode 100644 index 0000000..615d4d0 --- /dev/null +++ b/test/file-hash.test.ts @@ -0,0 +1,122 @@ +import { spawn } from "node:child_process"; +import fsSync from "node:fs"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { sha256File } from "../src/file-hash.js"; +import { configureFsSafeNative, __resetFsSafeNativeConfigForTest } from "../src/native-config.js"; +import { + __resetNativeLoaderForTest, + __setNativeLoaderForTest, + type NativeBinding, +} from "../src/native.js"; + +const tempDirs: string[] = []; + +async function tempRoot(): Promise { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "fs-safe-hash-")); + tempDirs.push(root); + return root; +} + +async function createFifo(filePath: string): Promise { + await new Promise((resolve, reject) => { + const child = spawn("mkfifo", [filePath]); + child.once("error", reject); + child.once("exit", (code) => { + if (code === 0) resolve(); + else reject(new Error(`mkfifo exited ${code}`)); + }); + }); +} + +afterEach(async () => { + vi.restoreAllMocks(); + __resetFsSafeNativeConfigForTest(); + __resetNativeLoaderForTest(); + await Promise.all(tempDirs.splice(0).map((root) => fs.rm(root, { recursive: true, force: true }))); +}); + +describe("sha256File", () => { + it("streams a path through the JavaScript fallback", async () => { + const root = await tempRoot(); + const filePath = path.join(root, "payload.bin"); + await fs.writeFile(filePath, "abc"); + configureFsSafeNative({ mode: "off" }); + + await expect(sha256File(filePath)).resolves.toEqual({ + bytes: 3, + digest: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + }); + }); + + it("does not close a caller-owned handle or change its current offset", async () => { + const root = await tempRoot(); + const filePath = path.join(root, "position.bin"); + await fs.writeFile(filePath, "abcdef"); + const handle = await fs.open(filePath, "r"); + configureFsSafeNative({ mode: "off" }); + try { + const prefix = Buffer.alloc(2); + await handle.read(prefix, 0, prefix.length, null); + await expect(sha256File(handle)).resolves.toMatchObject({ bytes: 6 }); + const next = Buffer.alloc(1); + await handle.read(next, 0, next.length, null); + expect(next.toString()).toBe("c"); + } finally { + await handle.close(); + } + }); + + it("uses the native async hash when the binding is available", async () => { + const root = await tempRoot(); + const filePath = path.join(root, "native.bin"); + await fs.writeFile(filePath, "native"); + const nativeHash = vi.fn(async () => ({ bytes: 6, digest: "native-digest" })); + __setNativeLoaderForTest( + () => ({ sha256File: nativeHash }) as unknown as NativeBinding, + ); + + await expect(sha256File(filePath)).resolves.toEqual({ + bytes: 6, + digest: "native-digest", + }); + expect(nativeHash).toHaveBeenCalledOnce(); + }); + + it.runIf(process.platform !== "win32")("rejects symbolic-link path inputs", async () => { + const root = await tempRoot(); + const filePath = path.join(root, "payload.bin"); + const linkPath = path.join(root, "link.bin"); + await fs.writeFile(filePath, "payload"); + await fs.symlink(filePath, linkPath); + + await expect(sha256File(linkPath)).rejects.toMatchObject({ code: "symlink" }); + }); + + it.runIf(process.platform !== "win32")( + "rejects a FIFO swapped in before open without blocking", + async () => { + const root = await tempRoot(); + const filePath = path.join(root, "payload.bin"); + const displacedPath = path.join(root, "payload.displaced"); + await fs.writeFile(filePath, "payload"); + configureFsSafeNative({ mode: "off" }); + const originalOpen = fs.open.bind(fs); + vi.spyOn(fs, "open").mockImplementation(async (candidate, flags, mode) => { + if (candidate === filePath) { + expect(typeof flags).toBe("number"); + expect((flags as number) & fsSync.constants.O_NONBLOCK).toBe( + fsSync.constants.O_NONBLOCK, + ); + await fs.rename(filePath, displacedPath); + await createFifo(filePath); + } + return await originalOpen(candidate, flags, mode); + }); + + await expect(sha256File(filePath)).rejects.toMatchObject({ code: "not-file" }); + }, + ); +}); diff --git a/test/findings-regression.test.ts b/test/findings-regression.test.ts index 9b0e21d..033d7f9 100644 --- a/test/findings-regression.test.ts +++ b/test/findings-regression.test.ts @@ -5,7 +5,7 @@ import path from "node:path"; import JSZip from "jszip"; import { afterEach, describe, expect, it, vi } from "vitest"; import { extractArchive } from "../src/archive.js"; -import { configureFsSafePython, root as openRoot } from "../src/index.js"; +import { configureFsSafeNative, root as openRoot } from "../src/index.js"; import { prepareArchiveDestinationDir, prepareArchiveOutputPath, mergeExtractedTreeIntoDestination } from "../src/archive-staging.js"; import { fileStore, fileStoreSync } from "../src/file-store.js"; import { writeJsonSync } from "../src/json.js"; @@ -40,14 +40,14 @@ async function writeOldFile(filePath: string, content = "old"): Promise { afterEach(async () => { vi.restoreAllMocks(); Object.defineProperty(process, "platform", originalPlatformDescriptor); - configureFsSafePython({ mode: "auto", pythonPath: undefined }); + configureFsSafeNative({ mode: "auto" }); __setFsSafeTestHooksForTest(undefined); await Promise.all(tempDirs.splice(0).map((dir) => fsp.rm(dir, { recursive: true, force: true }))); }); describe("security finding regressions", () => { it.runIf(process.platform !== "win32")("guards Root fallback mutators against parent swaps", async () => { - configureFsSafePython({ mode: "off" }); + configureFsSafeNative({ mode: "off" }); const base = await tempRoot("fs-safe-root-fallback-race-"); const outside = await tempRoot("fs-safe-root-fallback-outside-"); await fsp.mkdir(path.join(base, "nested")); @@ -147,7 +147,7 @@ describe("security finding regressions", () => { }); it.runIf(process.platform !== "win32")("uses unguessable no-follow temp files in pinned write fallback", async () => { - configureFsSafePython({ mode: "off" }); + configureFsSafeNative({ mode: "off" }); const base = await tempRoot("fs-safe-pinned-write-fallback-"); const outside = await tempRoot("fs-safe-pinned-write-outside-"); const outsideFile = path.join(outside, "outside.txt"); @@ -168,8 +168,8 @@ describe("security finding regressions", () => { await expect(fsp.readFile(outsideFile, "utf8")).resolves.toBe("outside"); }); - it("validates pinned write fallback payloads even when Python mode is off", async () => { - configureFsSafePython({ mode: "off" }); + it("validates pinned write fallback payloads even when native mode is off", async () => { + configureFsSafeNative({ mode: "off" }); const base = await tempRoot("fs-safe-pinned-write-validation-"); await expect( runPinnedWriteHelper({ diff --git a/test/fs-safe.test.ts b/test/fs-safe.test.ts index bc204a5..ce39515 100644 --- a/test/fs-safe.test.ts +++ b/test/fs-safe.test.ts @@ -3,7 +3,7 @@ import { chmod, mkdtemp, readdir, readFile, rename, rm, stat, symlink, writeFile import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { configureFsSafePython, FsSafeError, root as openRoot } from "../src/index.js"; +import { configureFsSafeNative, FsSafeError, root as openRoot } from "../src/index.js"; import { openLocalFileSafely, readLocalFileSafely } from "../src/root.js"; import { __setFsSafeTestHooksForTest } from "../src/test-hooks.js"; import { expectedFsSafeCode } from "./helpers/security.js"; @@ -19,7 +19,7 @@ async function tempRoot(prefix: string): Promise { } afterEach(async () => { - configureFsSafePython({ mode: "auto", pythonPath: undefined }); + configureFsSafeNative({ mode: "auto" }); __setFsSafeTestHooksForTest(undefined); const { rm } = await import("node:fs/promises"); await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { force: true, recursive: true }))); @@ -65,10 +65,10 @@ describe("@openclaw/fs-safe", () => { }); }); - it.skipIf(skipOnWindows)("can disable the Python helper while writes use best-effort fallbacks", async () => { - configureFsSafePython({ mode: "off" }); - const rootPath = await tempRoot("fs-safe-python-off-"); - const sourceRoot = await tempRoot("fs-safe-python-off-source-"); + it.skipIf(skipOnWindows)("can disable the native helper while writes use guarded fallbacks", async () => { + configureFsSafeNative({ mode: "off" }); + const rootPath = await tempRoot("fs-safe-native-off-"); + const sourceRoot = await tempRoot("fs-safe-native-off-source-"); const sourcePath = path.join(sourceRoot, "source.txt"); const root = await openRoot(rootPath); await writeFile(sourcePath, "copied"); @@ -355,38 +355,26 @@ describe("@openclaw/fs-safe", () => { await expect(readdir(rootPath)).resolves.toEqual([]); }); - it.runIf(process.platform !== "win32")("rejects pinned copy when the source path is swapped after identity capture", async () => { - const { runPinnedCopyHelper } = await import("../src/pinned-write.js"); + it.runIf(process.platform !== "win32")("rejects a copy when the source path is swapped after open", async () => { const rootPath = await tempRoot("fs-safe-copy-source-swap-root-"); const sourceRoot = await tempRoot("fs-safe-copy-source-swap-source-"); const sourcePath = path.join(sourceRoot, "source.txt"); const replacementPath = path.join(sourceRoot, "replacement.txt"); await writeFile(sourcePath, "original"); await writeFile(replacementPath, "replacement"); - const sourceIdentity = await stat(sourcePath); - await rm(sourcePath); - await rename(replacementPath, sourcePath); - - configureFsSafePython({ mode: "require" }); - try { - await runPinnedCopyHelper({ - rootPath, - relativeParentPath: "", - basename: "copied.txt", - mkdir: true, - mode: 0o600, - overwrite: true, - maxBytes: 1024, - sourcePath, - sourceIdentity: { dev: sourceIdentity.dev, ino: sourceIdentity.ino }, - }); - throw new Error("expected pinned copy source swap to fail"); - } catch (error) { - if (error instanceof FsSafeError && error.code === "helper-unavailable") { - return; - } - expect(error).toMatchObject({ code: "path-mismatch" }); - } + let swapped = false; + __setFsSafeTestHooksForTest({ + afterOpen: async (openedPath) => { + if (!swapped && openedPath === sourcePath) { + swapped = true; + await rm(sourcePath); + await rename(replacementPath, sourcePath); + } + }, + }); + const scoped = await openRoot(rootPath); + await expect(scoped.copyIn("copied.txt", sourcePath, { maxBytes: 1024 })).rejects + .toMatchObject({ code: "path-mismatch" }); await expect(stat(path.join(rootPath, "copied.txt"))).rejects.toMatchObject({ code: "ENOENT", }); diff --git a/test/guarded-mkdir-symlink-parent.test.ts b/test/guarded-mkdir-symlink-parent.test.ts index fdd826d..0c2dcc9 100644 --- a/test/guarded-mkdir-symlink-parent.test.ts +++ b/test/guarded-mkdir-symlink-parent.test.ts @@ -4,7 +4,7 @@ import path from "node:path"; import { afterEach, expect, it } from "vitest"; import { FsSafeError } from "../src/errors.js"; import { mkdirPathComponentsWithGuards } from "../src/guarded-mkdir.js"; -import { configureFsSafePython } from "../src/pinned-python-config.js"; +import { configureFsSafeNative } from "../src/native-config.js"; import { root } from "../src/root.js"; const tempDirs = new Set(); @@ -20,7 +20,7 @@ afterEach(async () => { await fs.rm(dir, { recursive: true, force: true }); } tempDirs.clear(); - configureFsSafePython({ mode: "auto" }); + configureFsSafeNative({ mode: "auto" }); }); // --- Unit-level coverage of mkdirPathComponentsWithGuards itself --- @@ -115,10 +115,12 @@ it("rejects a dangling symlink directory component with a typed FsSafeError, not // mkdirPathComponentsWithGuards returns, so the fix must flow the resolved // path back to the caller, not just resolve it internally. --- -it("writes a file through root().write() when the parent directory is an in-root symlink (mkdir: true)", async () => { +it.runIf(process.platform !== "win32")( + "writes a file through root().write() when the parent directory is an in-root symlink (mkdir: true)", + async () => { // Force the JS fallback path deterministically, matching this repo's own // convention in test/pinned-write-fallback-coverage.test.ts. - configureFsSafePython({ mode: "off" }); + configureFsSafeNative({ mode: "off" }); const rootDir = await tempRoot("fs-safe-root-write-symlink-"); const realDir = path.join(rootDir, "skills-bank", "skills", "auth-doctor"); @@ -139,4 +141,5 @@ it("writes a file through root().write() when the parent directory is an in-root const written = await fs.readFile(path.join(realDir, "SKILL.md"), "utf8"); expect(written).toBe("# new content\n"); -}); + }, +); diff --git a/test/guarded-write-cleanup.test.ts b/test/guarded-write-cleanup.test.ts index 2fc146a..cf12992 100644 --- a/test/guarded-write-cleanup.test.ts +++ b/test/guarded-write-cleanup.test.ts @@ -2,7 +2,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { configureFsSafePython } from "../src/pinned-python-config.js"; +import { configureFsSafeNative } from "../src/native-config.js"; import { runPinnedWriteHelper } from "../src/pinned-write.js"; const tempDirs: string[] = []; @@ -38,7 +38,7 @@ async function replaceParentAfterOpen(params: { afterEach(async () => { vi.restoreAllMocks(); - configureFsSafePython({ mode: "auto", pythonPath: undefined }); + configureFsSafeNative({ mode: "auto" }); Object.defineProperty(process, "platform", originalPlatformDescriptor); await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); }); @@ -48,7 +48,7 @@ const originalPlatformDescriptor = Object.getOwnPropertyDescriptor(process, "pla describe("guarded fallback write cleanup", () => { it.runIf(process.platform !== "win32")("closes pinned no-overwrite handles when post guards fail", async () => { Object.defineProperty(process, "platform", { configurable: true, value: "win32" }); - configureFsSafePython({ mode: "off" }); + configureFsSafeNative({ mode: "off" }); const base = await tempRoot("fs-safe-pinned-post-guard-"); const parentPath = path.join(base, "nested"); const movedParentPath = path.join(base, "nested-real"); diff --git a/test/move-path-regression.test.ts b/test/move-path-regression.test.ts index 13b4b99..97cf2aa 100644 --- a/test/move-path-regression.test.ts +++ b/test/move-path-regression.test.ts @@ -55,6 +55,122 @@ describe("movePathWithCopyFallback regressions", () => { ).toBeUndefined(); }); + it("rejects a hardlinked source before a same-filesystem rename", async () => { + const base = await tempRoot("fs-safe-move-rename-hardlink-"); + const source = path.join(base, "source.txt"); + const hardlink = path.join(base, "hardlink.txt"); + const dest = path.join(base, "dest.txt"); + await fsp.writeFile(source, "source"); + await fsp.link(source, hardlink); + const rename = vi.spyOn(fsp, "rename"); + + await expect( + movePathWithCopyFallback({ from: source, sourceHardlinks: "reject", to: dest }), + ).rejects.toMatchObject({ code: "hardlink" }); + + expect(rename).not.toHaveBeenCalled(); + await expect(fsp.readFile(source, "utf8")).resolves.toBe("source"); + await expect(fsp.readFile(hardlink, "utf8")).resolves.toBe("source"); + await expect(fsp.stat(dest)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("publishes a fresh inode when hardlink rejection is enabled", async () => { + const base = await tempRoot("fs-safe-move-reject-copy-"); + const source = path.join(base, "source.txt"); + const dest = path.join(base, "dest.txt"); + await fsp.writeFile(source, "source"); + const sourceIdentity = await fsp.stat(source); + const rename = vi.spyOn(fsp, "rename"); + + await movePathWithCopyFallback({ from: source, sourceHardlinks: "reject", to: dest }); + + expect(rename).not.toHaveBeenCalledWith(source, dest); + const targetIdentity = await fsp.stat(dest); + expect(targetIdentity.ino).not.toBe(sourceIdentity.ino); + await expect(fsp.readFile(dest, "utf8")).resolves.toBe("source"); + await expect(fsp.stat(source)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("rejects a hardlink added after preflight without publishing a target", async () => { + const base = await tempRoot("fs-safe-move-hardlink-race-"); + const source = path.join(base, "source.txt"); + const hardlink = path.join(base, "late-link.txt"); + const dest = path.join(base, "dest.txt"); + await fsp.writeFile(source, "source"); + const realLstat = fsp.lstat; + let sourceInspections = 0; + vi.spyOn(fsp, "lstat").mockImplementation(async (candidate, options) => { + const stat = await realLstat(candidate, options as never); + if (candidate === source && ++sourceInspections === 1) { + await fsp.link(source, hardlink); + } + return stat; + }); + + await expect( + movePathWithCopyFallback({ from: source, sourceHardlinks: "reject", to: dest }), + ).rejects.toMatchObject({ code: "hardlink" }); + + await expect(fsp.readFile(source, "utf8")).resolves.toBe("source"); + await expect(fsp.readFile(hardlink, "utf8")).resolves.toBe("source"); + await expect(fsp.stat(dest)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("rejects a directory move beneath itself before creating a staging tree", async () => { + const source = await tempRoot("fs-safe-move-self-descendant-"); + await fsp.writeFile(path.join(source, "payload.txt"), "payload"); + const dest = path.join(source, "child"); + + await expect( + movePathWithCopyFallback({ from: source, sourceHardlinks: "reject", to: dest }), + ).rejects.toMatchObject({ code: "invalid-path" }); + + await expect(fsp.readFile(path.join(source, "payload.txt"), "utf8")).resolves.toBe("payload"); + expect((await fsp.readdir(source)).toSorted()).toEqual(["payload.txt"]); + }); + + it("normalizes dot segments before rejecting a self-descendant move", async () => { + const source = await tempRoot("fs-safe-move-self-normalized-"); + await fsp.mkdir(path.join(source, "sub")); + await fsp.writeFile(path.join(source, "payload.txt"), "payload"); + const disguisedDest = path.join(source, "sub", "pivot", ".."); + + await expect( + movePathWithCopyFallback({ + from: source, + sourceHardlinks: "reject", + to: disguisedDest, + }), + ).rejects.toMatchObject({ code: "invalid-path" }); + + expect((await fsp.readdir(source)).toSorted()).toEqual(["payload.txt", "sub"]); + }); + + it.runIf(process.platform !== "win32")( + "rejects a self-descendant copy reached through a symlinked parent", + async () => { + const base = await tempRoot("fs-safe-move-self-symlink-"); + const source = path.join(base, "source"); + const alias = path.join(base, "alias"); + await fsp.mkdir(source); + await fsp.writeFile(path.join(source, "payload.txt"), "payload"); + await fsp.symlink(source, alias, "dir"); + + await expect( + movePathWithCopyFallback({ + from: source, + sourceHardlinks: "reject", + to: path.join(alias, "child"), + }), + ).rejects.toMatchObject({ code: "invalid-path" }); + + await expect(fsp.readFile(path.join(source, "payload.txt"), "utf8")).resolves.toBe( + "payload", + ); + expect((await fsp.readdir(source)).toSorted()).toEqual(["payload.txt"]); + }, + ); + it.runIf(process.platform !== "win32")( "does not delete source entries replaced after an EXDEV copy", async () => { diff --git a/test/native-archive-equivalence.test.ts b/test/native-archive-equivalence.test.ts new file mode 100644 index 0000000..a609c2b --- /dev/null +++ b/test/native-archive-equivalence.test.ts @@ -0,0 +1,424 @@ +import { createRequire } from "node:module"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + ARCHIVE_LIMIT_ERROR_CODE, + extractArchive, + readArchiveEntry, + resolveArchiveKind, + ArchiveFormatError, + ArchiveLimitError, +} from "../src/archive.js"; +import { configureFsSafeNative, __resetFsSafeNativeConfigForTest } from "../src/native-config.js"; +import { + __resetNativeLoaderForTest, + __setNativeLoaderForTest, + type NativeBinding, +} from "../src/native.js"; + +const require = createRequire(import.meta.url); +let native: NativeBinding | undefined; +try { + native = require("../native") as NativeBinding; +} catch { + // JS-only jobs intentionally exercise the fallback without a built binding. +} +const tempDirs: string[] = []; + +type TarFixtureEntry = { + path: string; + body?: string; + mode?: number; + type?: "0" | "1" | "2" | "5" | "7" | "K" | "L" | "S" | "g" | "x"; + linkPath?: string; + base256Size?: number; +}; + +function writeString(block: Buffer, offset: number, length: number, value: string): void { + block.write(value, offset, Math.min(length, Buffer.byteLength(value)), "utf8"); +} + +function writeOctal(block: Buffer, offset: number, length: number, value: number): void { + writeString(block, offset, length, `${value.toString(8).padStart(length - 1, "0")}\0`); +} + +function tarFixture(entries: TarFixtureEntry[]): Buffer { + const blocks: Buffer[] = []; + for (const fixture of entries) { + const body = Buffer.from(fixture.body ?? ""); + const type = fixture.type ?? "0"; + const header = Buffer.alloc(512); + writeString(header, 0, 100, fixture.path); + writeOctal(header, 100, 8, fixture.mode ?? (type === "5" ? 0o755 : 0o644)); + writeOctal(header, 108, 8, 0); + writeOctal(header, 116, 8, 0); + const hasBody = + type === "0" || + type === "7" || + type === "K" || + type === "L" || + type === "g" || + type === "x"; + const size = fixture.base256Size ?? (hasBody ? body.length : 0); + if (fixture.base256Size === undefined) { + writeOctal(header, 124, 12, size); + } else { + header[124] = 0x80; + header.writeBigUInt64BE(BigInt(size), 128); + } + writeOctal(header, 136, 12, 0); + header.fill(0x20, 148, 156); + writeString(header, 156, 1, type); + writeString(header, 157, 100, fixture.linkPath ?? ""); + writeString(header, 257, 6, "ustar\0"); + writeString(header, 263, 2, "00"); + const checksum = header.reduce((sum, byte) => sum + byte, 0); + writeString(header, 148, 8, `${checksum.toString(8).padStart(6, "0")}\0 `); + blocks.push(header); + if (hasBody && fixture.base256Size === undefined) { + blocks.push(body, Buffer.alloc((512 - (body.length % 512)) % 512)); + } + } + blocks.push(Buffer.alloc(1024)); + return Buffer.concat(blocks); +} + +async function tempRoot(): Promise { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "fs-safe-native-archive-")); + tempDirs.push(root); + return root; +} + +function useBackend(backend: "native" | "javascript"): void { + if (backend === "native") { + __setNativeLoaderForTest(() => native!); + configureFsSafeNative({ mode: "require" }); + } else { + configureFsSafeNative({ mode: "off" }); + } +} + +async function settleWithin(promise: Promise, milliseconds = 2_000): Promise { + let timeout: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + promise, + new Promise((_resolve, reject) => { + timeout = setTimeout( + () => reject(new Error(`archive rejection did not settle within ${milliseconds}ms`)), + milliseconds, + ); + }), + ]); + } finally { + if (timeout) clearTimeout(timeout); + } +} + +afterEach(async () => { + __resetFsSafeNativeConfigForTest(); + __resetNativeLoaderForTest(); + await Promise.all(tempDirs.splice(0).map((root) => fs.rm(root, { recursive: true, force: true }))); +}); + +const archiveBackends = native + ? (["native", "javascript"] as const) + : (["javascript"] as const); + +describe.each(archiveBackends)("%s archive path", (backend) => { + it("extracts and reads the same clamped regular file", async () => { + useBackend(backend); + const root = await tempRoot(); + const archivePath = path.join(root, "fixture.tar"); + const destination = path.join(root, "destination"); + await fs.writeFile(archivePath, tarFixture([{ path: "bin/tool", body: "payload", mode: 0o7777 }])); + await fs.mkdir(destination); + + await extractArchive({ archivePath, destDir: destination, timeoutMs: 10_000 }); + await expect(fs.readFile(path.join(destination, "bin", "tool"), "utf8")).resolves.toBe("payload"); + await expect(readArchiveEntry(archivePath, "bin/tool", { maxBytes: 7 })).resolves.toEqual(Buffer.from("payload")); + if (process.platform !== "win32") { + expect((await fs.stat(path.join(destination, "bin", "tool"))).mode & 0o7777).toBe(0o755); + } + }); + + it("normalizes a dot-prefixed TAR member for bounded reads", async () => { + useBackend(backend); + const root = await tempRoot(); + const archivePath = path.join(root, "dot-path.tar"); + await fs.writeFile(archivePath, tarFixture([{ path: "./value.txt", body: "value" }])); + await expect(readArchiveEntry(archivePath, "value.txt", { maxBytes: 5 })).resolves.toEqual( + Buffer.from("value"), + ); + }); + + it("rejects traversal, symbolic links, and hard links", async () => { + useBackend(backend); + for (const [name, entry] of [ + ["traversal", { path: "../escape", body: "owned" }], + ["symlink", { path: "link", type: "2" as const, linkPath: "../escape" }], + ["hardlink", { path: "link", type: "1" as const, linkPath: "target" }], + ] as const) { + const root = await tempRoot(); + const archivePath = path.join(root, `${name}.tar`); + const destination = path.join(root, "destination"); + await fs.writeFile(archivePath, tarFixture([entry])); + await fs.mkdir(destination); + await expect( + extractArchive({ archivePath, destDir: destination, timeoutMs: 10_000 }), + ).rejects.toBeTruthy(); + await expect(fs.readdir(destination)).resolves.toEqual([]); + } + }); + + it("settles every TAR policy rejection without leaving the parser paused", async () => { + useBackend(backend); + const cases = [ + { + name: "filtered-symlink", + entry: { path: "fleet/link", type: "2" as const, linkPath: "../outside" }, + options: { + entryFilter: (entry: { kind: string }) => + entry.kind === "symlink" ? ("skip" as const) : ("extract" as const), + }, + expected: { name: "ArchiveSecurityError", code: "entry-filtered" }, + }, + { + name: "blocked-symlink", + entry: { path: "fleet/link", type: "2" as const, linkPath: "../outside" }, + options: {}, + expected: { name: "ArchiveSecurityError", code: "entry-link" }, + }, + { + name: "traversal", + entry: { path: "../outside", body: "owned" }, + options: {}, + expected: { name: "ArchiveSecurityError", code: "entry-path" }, + }, + { + name: "entry-limit", + entry: { path: "oversized", body: "too large" }, + options: { limits: { maxEntryBytes: 1 } }, + expected: { code: ARCHIVE_LIMIT_ERROR_CODE.ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT }, + }, + ]; + + for (const fixture of cases) { + const root = await tempRoot(); + const archivePath = path.join(root, `${fixture.name}.tar`); + const destination = path.join(root, "destination"); + await fs.writeFile(archivePath, tarFixture([fixture.entry])); + await fs.mkdir(destination); + + await expect( + settleWithin( + extractArchive({ + archivePath, + destDir: destination, + timeoutMs: 10_000, + ...fixture.options, + }), + ), + ).rejects.toMatchObject(fixture.expected); + await expect(fs.readdir(destination)).resolves.toEqual([]); + } + }); + + it("enforces entry-count, per-entry, and total byte budgets", async () => { + useBackend(backend); + const fixture = tarFixture([ + { path: "one", body: "1234" }, + { path: "two", body: "5678" }, + ]); + for (const [limits, code] of [ + [{ maxEntries: 1 }, ARCHIVE_LIMIT_ERROR_CODE.ENTRY_COUNT_EXCEEDS_LIMIT], + [{ maxEntryBytes: 3 }, ARCHIVE_LIMIT_ERROR_CODE.ENTRY_EXTRACTED_SIZE_EXCEEDS_LIMIT], + [{ maxExtractedBytes: 7 }, ARCHIVE_LIMIT_ERROR_CODE.EXTRACTED_SIZE_EXCEEDS_LIMIT], + ] as const) { + const root = await tempRoot(); + const archivePath = path.join(root, "limits.tar"); + const destination = path.join(root, "destination"); + await fs.writeFile(archivePath, fixture); + await fs.mkdir(destination); + await expect( + extractArchive({ archivePath, destDir: destination, timeoutMs: 10_000, limits }), + ).rejects.toMatchObject({ code }); + await expect(fs.readdir(destination)).resolves.toEqual([]); + } + }); + + it("rejects deep entry paths before creating implicit directories", async () => { + useBackend(backend); + const root = await tempRoot(); + const archivePath = path.join(root, "deep-path.tar"); + const destination = path.join(root, "destination"); + await fs.writeFile( + archivePath, + tarFixture([{ path: "one/two/three/four/value.txt", body: "payload" }]), + ); + await fs.mkdir(destination); + + await expect( + extractArchive({ + archivePath, + destDir: destination, + timeoutMs: 10_000, + limits: { maxEntryPathComponents: 4 }, + }), + ).rejects.toMatchObject({ + code: ARCHIVE_LIMIT_ERROR_CODE.ENTRY_PATH_COMPONENTS_EXCEEDS_LIMIT, + }); + await expect(fs.readdir(destination)).resolves.toEqual([]); + }); + + it("applies byte budgets after explicitly skipped entries", async () => { + useBackend(backend); + const root = await tempRoot(); + const archivePath = path.join(root, "filtered-limits.tar"); + const destination = path.join(root, "destination"); + await fs.writeFile( + archivePath, + tarFixture([ + { path: "skip", body: "oversized" }, + { path: "keep", body: "k" }, + ]), + ); + await fs.mkdir(destination); + + await extractArchive({ + archivePath, + destDir: destination, + timeoutMs: 10_000, + limits: { maxEntryBytes: 1, maxExtractedBytes: 1 }, + entryFilter: (entry) => (entry.path === "skip" ? "skip" : "extract"), + onFiltered: "skip-entry", + }); + await expect(fs.readFile(path.join(destination, "keep"), "utf8")).resolves.toBe("k"); + }); + + it.each([ + ["oversized PAX", [{ path: "PaxHeader", type: "x" as const, body: "path=very-long-name\n" }]], + [ + "oversized GNU longname", + [{ path: "LongName", type: "L" as const, body: "this-name-is-definitely-long\0" }], + ], + [ + "chained metadata", + [ + { path: "LongName", type: "L" as const, body: "short\0" }, + { path: "LongLink", type: "K" as const, body: "oversized-link-target\0" }, + ], + ], + [ + "base-256 metadata size", + [{ path: "PaxHeader", type: "x" as const, base256Size: 17 }], + ], + ])("rejects %s before metadata buffering", async (_label, entries) => { + useBackend(backend); + const root = await tempRoot(); + const archivePath = path.join(root, "metadata.tar"); + const destination = path.join(root, "destination"); + await fs.writeFile(archivePath, tarFixture(entries)); + await fs.mkdir(destination); + + await expect( + extractArchive({ + archivePath, + destDir: destination, + timeoutMs: 10_000, + limits: { maxMetaEntryBytes: 16 }, + }), + ).rejects.toSatisfy((error: unknown) => { + expect(error).toBeInstanceOf(ArchiveLimitError); + expect(error).toMatchObject({ + code: ARCHIVE_LIMIT_ERROR_CODE.META_ENTRY_SIZE_EXCEEDS_LIMIT, + }); + return true; + }); + await expect(fs.readdir(destination)).resolves.toEqual([]); + }); + + it("rejects a truncated TAR header with the same typed format error", async () => { + useBackend(backend); + const root = await tempRoot(); + const archivePath = path.join(root, "truncated.tar"); + const destination = path.join(root, "destination"); + const truncated = tarFixture([{ path: "value", body: "value" }]).subarray(0, 511); + await fs.writeFile(archivePath, truncated); + await fs.mkdir(destination); + + await expect( + extractArchive({ archivePath, destDir: destination, timeoutMs: 10_000 }), + ).rejects.toSatisfy((error: unknown) => { + expect(error).toBeInstanceOf(ArchiveFormatError); + expect(error).toMatchObject({ code: "archive-header-invalid" }); + return true; + }); + }); + + it.each([ + ["PAX size overrides", { path: "PaxHeader", type: "x" as const, body: "10 path=a\n" }], + ["GNU sparse entries", { path: "sparse", type: "S" as const }], + ])("rejects unmeterable %s with the same typed format error", async (_label, entry) => { + useBackend(backend); + const root = await tempRoot(); + const archivePath = path.join(root, "unmeterable.tar"); + const destination = path.join(root, "destination"); + await fs.writeFile(archivePath, tarFixture([entry])); + await fs.mkdir(destination); + + await expect( + extractArchive({ archivePath, destDir: destination, timeoutMs: 10_000 }), + ).rejects.toMatchObject({ code: "archive-header-invalid" }); + }); + + it("treats contiguous entries as regular files", async () => { + useBackend(backend); + const root = await tempRoot(); + const archivePath = path.join(root, "contiguous.tar"); + const destination = path.join(root, "destination"); + await fs.writeFile( + archivePath, + tarFixture([{ path: "contiguous", type: "7", body: "payload" }]), + ); + await fs.mkdir(destination); + + await extractArchive({ archivePath, destDir: destination, timeoutMs: 10_000 }); + await expect(fs.readFile(path.join(destination, "contiguous"), "utf8")).resolves.toBe( + "payload", + ); + }); +}); + +describe.runIf(Boolean(native))("native-only compressed tar formats", () => { + const fixtures = { + "tar-bzip2": "QlpoOTFBWSZTWR7OLWUAAE57kNIABIBAA3+AAIBuZt/ABAAgCCAAciIT1MmhkDQNAaeSCVNTyKeU9Qaek8oB6h6grzvOX5w+ADqMF1cEAQkPSi8X9JSUNEAhmhZFEfFXVrk06WnAZd6xiqSZl1ns0+55YMXVrY+KHgFL4hZYy28xFJSAznPLVCPxdyRThQkB7OLWUA==", + "tar-zstd": "KLUv/WQAB7UDADKFEReQpzpAWzCQC1aaeGamglLuJoOiujuRBIXgqmerYAic+geI7xfhq/ZgabX5RhoV9CE0pyAWcvBMbNvORGdM2h6bWMCbSocRAPGAHwKkUFkZAg5E65ccFUDlwxo5gAxqHgpO4NMsGGCrmDkAOXBuxdo3ASQjp+s0", + } as const; + + for (const [kind, base64] of Object.entries(fixtures) as Array<[keyof typeof fixtures, string]>) { + it(`extracts and reads ${kind}`, async () => { + useBackend("native"); + const root = await tempRoot(); + const extension = kind === "tar-zstd" ? "tar.zst" : "tar.bz2"; + const archivePath = path.join(root, `fixture.${extension}`); + const destination = path.join(root, "destination"); + await fs.writeFile(archivePath, Buffer.from(base64, "base64")); + await fs.mkdir(destination); + + expect(resolveArchiveKind(archivePath)).toBe(kind); + await extractArchive({ archivePath, destDir: destination, timeoutMs: 10_000 }); + await expect(fs.readFile(path.join(destination, "value.txt"), "utf8")).resolves.toBe("compressed-value"); + await expect(readArchiveEntry(archivePath, "value.txt", { maxBytes: 16 })).resolves.toEqual(Buffer.from("compressed-value")); + }); + } + + it("reports a typed actionable error when a native-only format is forced off", async () => { + configureFsSafeNative({ mode: "off" }); + expect(() => resolveArchiveKind("fixture.tar.zst")).toThrow( + expect.objectContaining({ code: "helper-unavailable" }), + ); + }); +}); diff --git a/test/native-integration.test.ts b/test/native-integration.test.ts new file mode 100644 index 0000000..61da426 --- /dev/null +++ b/test/native-integration.test.ts @@ -0,0 +1,176 @@ +import fsSync from "node:fs"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { createRequire } from "node:module"; +import { afterEach, describe, expect, it } from "vitest"; +import { configureFsSafeNative } from "../src/native-config.js"; +import { acquireFileLock } from "../src/file-lock.js"; +import { __resetNativeLoaderForTest, __setNativeLoaderForTest } from "../src/native.js"; +import { publishFileExclusive } from "../src/publish-file.js"; +import { runPinnedWriteHelper } from "../src/pinned-write.js"; + +type NativeBinding = typeof import("../native/index.js"); + +const require = createRequire(import.meta.url); +let native: NativeBinding | undefined; +try { + native = require("../native") as NativeBinding; +} catch { + // Native artifacts are built by dedicated platform jobs. The ordinary JS + // matrix deliberately proves that installation without them still works. +} + +const roots: string[] = []; + +afterEach(async () => { + configureFsSafeNative({ mode: "auto" }); + __resetNativeLoaderForTest(); + await Promise.all(roots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true }))); +}); + +describe.runIf(native)("native filesystem primitives", () => { + it("opens beneath a directory descriptor and reports fd identity", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "fs-safe-native-open-")); + roots.push(root); + await fs.mkdir(path.join(root, "nested")); + await fs.writeFile(path.join(root, "nested", "value"), "ok"); + const rootFd = fsSync.openSync(root, fsSync.constants.O_RDONLY); + try { + const fd = native!.openBeneath(rootFd, "nested/value", fsSync.constants.O_RDONLY); + try { + expect(native!.fstatIdentity(fd)).toMatchObject({ isFile: true, size: 2 }); + } finally { + fsSync.closeSync(fd); + } + expect(() => native!.openBeneath(rootFd, "../outside", fsSync.constants.O_RDONLY)).toThrow(); + } finally { + fsSync.closeSync(rootFd); + } + }); + + it("maps no-replace collisions to EEXIST without changing either file", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "fs-safe-native-rename-")); + roots.push(root); + await fs.writeFile(path.join(root, "source"), "source"); + await fs.writeFile(path.join(root, "target"), "target"); + const rootFd = fsSync.openSync(root, fsSync.constants.O_RDONLY); + try { + expect(() => native!.renameNoReplace(rootFd, "source", rootFd, "target")).toThrowError( + expect.objectContaining({ code: "EEXIST" }), + ); + } finally { + fsSync.closeSync(rootFd); + } + await expect(fs.readFile(path.join(root, "source"), "utf8")).resolves.toBe("source"); + await expect(fs.readFile(path.join(root, "target"), "utf8")).resolves.toBe("target"); + }); + + it.runIf(process.platform === "win32")("rejects reparse-point directory components", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "fs-safe-native-reparse-")); + roots.push(root); + const real = path.join(root, "real"); + await fs.mkdir(real); + await fs.writeFile(path.join(real, "value"), "ok"); + await fs.symlink(real, path.join(root, "alias"), "junction"); + const rootFd = fsSync.openSync(root, fsSync.constants.O_RDONLY); + try { + expect(() => native!.openBeneath(rootFd, "alias/value", fsSync.constants.O_RDONLY)) + .toThrow(); + } finally { + fsSync.closeSync(rootFd); + } + }); + + it("uses native no-replace commits for create-only pinned writes", async () => { + let renameCalls = 0; + __setNativeLoaderForTest(() => ({ + ...native!, + renameNoReplace(...args) { + renameCalls += 1; + return native!.renameNoReplace(...args); + }, + })); + configureFsSafeNative({ mode: "require" }); + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "fs-safe-native-write-")); + roots.push(directory); + await runPinnedWriteHelper({ + rootPath: directory, + relativeParentPath: "nested", + basename: "value", + mkdir: true, + mode: 0o600, + overwrite: false, + input: { kind: "buffer", data: "native" }, + }); + expect(renameCalls).toBe(1); + await expect(fs.readFile(path.join(directory, "nested/value"), "utf8")).resolves.toBe("native"); + }); + + it("rejects native writes when the expected root identity does not match", async () => { + __setNativeLoaderForTest(() => native!); + configureFsSafeNative({ mode: "require" }); + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "fs-safe-native-identity-")); + roots.push(directory); + const identity = await fs.lstat(directory); + await expect(runPinnedWriteHelper({ + rootPath: directory, + relativeParentPath: "", + basename: "value", + mkdir: false, + mode: 0o600, + overwrite: false, + input: { kind: "buffer", data: "value" }, + rootIdentity: { dev: identity.dev + 1, ino: identity.ino }, + })).rejects.toMatchObject({ code: "path-mismatch" }); + }); + + it("creates sidecar locks through native exclusive open", async () => { + let exclusiveOpenCalls = 0; + __setNativeLoaderForTest(() => ({ + ...native!, + openBeneath(rootFd, relPath, flags) { + if (flags & fsSync.constants.O_EXCL) { + exclusiveOpenCalls += 1; + } + return native!.openBeneath(rootFd, relPath, flags); + }, + })); + configureFsSafeNative({ mode: "require" }); + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "fs-safe-native-lock-")); + roots.push(directory); + const targetPath = path.join(directory, "state.json"); + const lock = await acquireFileLock(targetPath, { payload: () => ({ pid: process.pid }) }); + try { + expect(exclusiveOpenCalls).toBe(1); + await expect(lock.verifyStillHeld()).resolves.toBe(true); + } finally { + await lock.release(); + } + }); + + it("publishes by native rename without replacing an existing target", async () => { + __setNativeLoaderForTest(() => native!); + configureFsSafeNative({ mode: "require" }); + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "fs-safe-native-publish-")); + roots.push(directory); + const sourcePath = path.join(directory, "source"); + const targetPath = path.join(directory, "target"); + await fs.writeFile(sourcePath, "content"); + const result = await publishFileExclusive({ + sourcePath, + targetPath, + strategy: "rename-noreplace", + }); + expect(result.method).toBe("rename-noreplace"); + await expect(fs.readFile(targetPath, "utf8")).resolves.toBe("content"); + await expect(fs.lstat(sourcePath)).rejects.toMatchObject({ code: "ENOENT" }); + + await fs.writeFile(sourcePath, "second"); + await expect( + publishFileExclusive({ sourcePath, targetPath, strategy: "rename-noreplace" }), + ).rejects.toMatchObject({ code: "EEXIST" }); + await expect(fs.readFile(sourcePath, "utf8")).resolves.toBe("second"); + await expect(fs.readFile(targetPath, "utf8")).resolves.toBe("content"); + }); +}); diff --git a/test/native-loader.test.ts b/test/native-loader.test.ts new file mode 100644 index 0000000..3fc404c --- /dev/null +++ b/test/native-loader.test.ts @@ -0,0 +1,118 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + __resetFsSafeNativeConfigForTest, + configureFsSafePython, + configureFsSafeNative, + getFsSafeNativeConfig, +} from "../src/native-config.js"; +import { + __resetNativeLoaderForTest, + __setNativeLoaderForTest, + getNativeBinding, + type NativeBinding, +} from "../src/native.js"; + +const envKeys = [ + "FS_SAFE_NATIVE_MODE", + "OPENCLAW_FS_SAFE_NATIVE_MODE", + "FS_SAFE_PYTHON_MODE", + "OPENCLAW_FS_SAFE_PYTHON_MODE", + "FS_SAFE_PYTHON", + "OPENCLAW_FS_SAFE_PYTHON", + "OPENCLAW_PINNED_PYTHON", + "OPENCLAW_PINNED_WRITE_PYTHON", +] as const; +const originalEnv = Object.fromEntries(envKeys.map((key) => [key, process.env[key]])); + +afterEach(() => { + vi.restoreAllMocks(); + __resetFsSafeNativeConfigForTest(); + __resetNativeLoaderForTest(); + for (const key of envKeys) { + const value = originalEnv[key]; + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } +}); + +describe("native helper configuration", () => { + it("reads the environment and lets programmatic configuration win", () => { + process.env.FS_SAFE_NATIVE_MODE = "off"; + expect(getFsSafeNativeConfig()).toEqual({ mode: "off" }); + configureFsSafeNative({ mode: "require" }); + expect(getFsSafeNativeConfig()).toEqual({ mode: "require" }); + }); + + it("accepts documented boolean and compatibility mode spellings", () => { + process.env.FS_SAFE_NATIVE_MODE = "required"; + expect(getFsSafeNativeConfig()).toEqual({ mode: "require" }); + process.env.FS_SAFE_NATIVE_MODE = "never"; + expect(getFsSafeNativeConfig()).toEqual({ mode: "off" }); + process.env.FS_SAFE_NATIVE_MODE = "true"; + expect(getFsSafeNativeConfig()).toEqual({ mode: "auto" }); + }); + + it("warns once and maps legacy Python configuration during the 0.5 migration", () => { + const emitWarning = vi.spyOn(process, "emitWarning").mockImplementation(() => undefined); + process.env.FS_SAFE_PYTHON_MODE = "required"; + process.env.FS_SAFE_PYTHON = "/legacy/python"; + + expect(getFsSafeNativeConfig()).toEqual({ mode: "require" }); + expect(getFsSafeNativeConfig()).toEqual({ mode: "require" }); + expect(emitWarning).toHaveBeenCalledTimes(1); + expect(emitWarning).toHaveBeenCalledWith( + expect.stringContaining('mapped to native mode "require"'), + expect.objectContaining({ code: "FS_SAFE_PYTHON_DEPRECATED" }), + ); + }); + + it("keeps configureFsSafePython as a warning migration bridge only for 0.5", () => { + const emitWarning = vi.spyOn(process, "emitWarning").mockImplementation(() => undefined); + configureFsSafePython({ mode: "off", pythonPath: "/legacy/python" }); + + expect(getFsSafeNativeConfig()).toEqual({ mode: "off" }); + expect(emitWarning).toHaveBeenCalledTimes(1); + expect(emitWarning).toHaveBeenCalledWith( + expect.stringContaining("configureFsSafeNative"), + expect.objectContaining({ code: "FS_SAFE_PYTHON_DEPRECATED" }), + ); + }); + + it("falls back in auto mode and fails closed in require mode", () => { + const unavailable = vi.fn(() => { + throw Object.assign(new Error("missing binding"), { code: "MODULE_NOT_FOUND" }); + }); + __setNativeLoaderForTest(unavailable); + configureFsSafeNative({ mode: "auto" }); + expect(getNativeBinding()).toBeUndefined(); + expect(unavailable).toHaveBeenCalledTimes(1); + + configureFsSafeNative({ mode: "require" }); + expect(() => getNativeBinding()).toThrowError( + expect.objectContaining({ code: "helper-unavailable" }), + ); + expect(unavailable).toHaveBeenCalledTimes(1); + }); + + it("does not attempt to load the binding in off mode", () => { + const loader = vi.fn(() => ({}) as NativeBinding); + __setNativeLoaderForTest(loader); + configureFsSafeNative({ mode: "off" }); + expect(getNativeBinding()).toBeUndefined(); + expect(loader).not.toHaveBeenCalled(); + }); +}); + +describe("native package loader", () => { + it("contains no import-time process execution path", () => { + const loader = readFileSync(fileURLToPath(new URL("../native/index.js", import.meta.url)), "utf8"); + expect(loader).not.toMatch(/(?:child_process|execSync|execFileSync|spawnSync|\bspawn\s*\()/); + expect(loader).toContain("isMuslFromElfInterpreter"); + expect(loader).toContain("Unknown Linux libc: try the glibc package"); + }); +}); diff --git a/test/native-publish-equivalence.test.ts b/test/native-publish-equivalence.test.ts new file mode 100644 index 0000000..b3e4eed --- /dev/null +++ b/test/native-publish-equivalence.test.ts @@ -0,0 +1,249 @@ +import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import fsSync from "node:fs"; +import { createRequire } from "node:module"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { configureFsSafeNative, __resetFsSafeNativeConfigForTest } from "../src/native-config.js"; +import { + __resetNativeLoaderForTest, + __setNativeLoaderForTest, + type NativeBinding, +} from "../src/native.js"; +import { publishFileExclusive } from "../src/publish-file.js"; + +const require = createRequire(import.meta.url); +let native: NativeBinding | undefined; +try { + native = require("../native") as NativeBinding; +} catch { + // JS-only jobs intentionally exercise the fallback without a built binding. +} +const tempDirs: string[] = []; + +async function tempRoot(): Promise { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "fs-safe-native-publish-")); + tempDirs.push(root); + return root; +} + +afterEach(async () => { + vi.restoreAllMocks(); + __resetFsSafeNativeConfigForTest(); + __resetNativeLoaderForTest(); + await Promise.all(tempDirs.splice(0).map((root) => fs.rm(root, { recursive: true, force: true }))); +}); + +describe.runIf(Boolean(native))("native publication primitives", () => { + it("hashes the same bytes as Node without changing the descriptor position", async () => { + const root = await tempRoot(); + const sourcePath = path.join(root, "source"); + const payload = Buffer.from("native hash equivalence".repeat(4096)); + await fs.writeFile(sourcePath, payload); + const source = await fs.open(sourcePath, "r"); + try { + const before = await source.read(Buffer.alloc(1), 0, 1, null); + const digest = await native!.sha256File(source.fd); + const after = await source.read(Buffer.alloc(1), 0, 1, null); + expect(digest).toEqual({ + bytes: payload.length, + digest: createHash("sha256").update(payload).digest("hex"), + }); + expect(before.buffer[0]).toBe(payload[0]); + expect(after.buffer[0]).toBe(payload[1]); + } finally { + await source.close(); + } + }); + + it("clones exclusively and preserves an existing target", async () => { + const root = await tempRoot(); + const sourcePath = path.join(root, "source"); + const targetPath = path.join(root, "target"); + await fs.writeFile(sourcePath, "clone-payload"); + const source = await fs.open(sourcePath, "r"); + const directory = await fs.open(root, fsSync.constants.O_RDONLY | fsSync.constants.O_DIRECTORY); + try { + let clonedFd: number; + try { + clonedFd = native!.cloneFileExclusive(source.fd, directory.fd, "target"); + } catch (error) { + if (process.platform === "darwin") throw error; + expect(error).toMatchObject({ code: "ENOTSUP" }); + return; + } + fsSync.closeSync(clonedFd); + await expect(fs.readFile(targetPath, "utf8")).resolves.toBe("clone-payload"); + await expect(() => native!.cloneFileExclusive(source.fd, directory.fd, "target")).toThrow( + expect.objectContaining({ code: "EEXIST" }), + ); + await expect(fs.readFile(targetPath, "utf8")).resolves.toBe("clone-payload"); + } finally { + await source.close(); + await directory.close(); + } + }); + + it.runIf(process.platform === "darwin")( + "clones APFS files with xattrs and strips custom metadata from the target", + async () => { + const root = await tempRoot(); + const sourcePath = path.join(root, "source-xattr"); + const targetPath = path.join(root, "target-xattr"); + await fs.writeFile(sourcePath, "clone-with-xattr"); + execFileSync("xattr", ["-w", "com.openclaw.fs-safe-test", "fixture", sourcePath]); + const source = await fs.open(sourcePath, "r"); + const directory = await fs.open( + root, + fsSync.constants.O_RDONLY | fsSync.constants.O_DIRECTORY, + ); + try { + const clonedFd = native!.cloneFileExclusive(source.fd, directory.fd, "target-xattr"); + fsSync.closeSync(clonedFd); + } finally { + await source.close(); + await directory.close(); + } + await expect(fs.readFile(targetPath, "utf8")).resolves.toBe("clone-with-xattr"); + expect(execFileSync("xattr", [targetPath], { encoding: "utf8" })).not.toContain( + "com.openclaw.fs-safe-test", + ); + }, + ); + + it("fences a native clone result by identity and hash", async () => { + const root = await tempRoot(); + const sourcePath = path.join(root, "source-fenced"); + const targetPath = path.join(root, "target-fenced"); + await fs.writeFile(sourcePath, "clone-fenced-payload"); + __setNativeLoaderForTest(() => ({ + ...native!, + linkBeneath() { + throw Object.assign(new Error("force clone"), { code: "EXDEV" }); + }, + cloneFileExclusive() { + fsSync.copyFileSync(sourcePath, targetPath, fsSync.constants.COPYFILE_EXCL); + fsSync.chmodSync(targetPath, 0o600); + return fsSync.openSync(targetPath, "r+"); + }, + })); + configureFsSafeNative({ mode: "require" }); + + const result = await publishFileExclusive({ + sourcePath, + targetPath, + strategy: "link-or-copy", + }); + expect(result.method).toBe("exclusive-copy"); + await expect(fs.readFile(targetPath, "utf8")).resolves.toBe("clone-fenced-payload"); + }); + + it("removes a native clone when post-copy hash fencing fails", async () => { + const root = await tempRoot(); + const sourcePath = path.join(root, "source-mismatch"); + const targetPath = path.join(root, "target-mismatch"); + await fs.writeFile(sourcePath, "clone-mismatch-payload"); + let hashCalls = 0; + __setNativeLoaderForTest(() => ({ + ...native!, + linkBeneath() { + throw Object.assign(new Error("force clone"), { code: "EXDEV" }); + }, + cloneFileExclusive() { + fsSync.copyFileSync(sourcePath, targetPath, fsSync.constants.COPYFILE_EXCL); + fsSync.chmodSync(targetPath, 0o600); + return fsSync.openSync(targetPath, "r+"); + }, + async sha256File(fd) { + const result = await native!.sha256File(fd); + hashCalls += 1; + return hashCalls === 2 ? { ...result, digest: "0".repeat(64) } : result; + }, + })); + configureFsSafeNative({ mode: "require" }); + + await expect( + publishFileExclusive({ sourcePath, targetPath, strategy: "link-or-copy" }), + ).rejects.toMatchObject({ code: "path-mismatch" }); + await expect(fs.access(targetPath)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it.runIf(process.platform === "linux")("copies exclusively with copy_file_range", async () => { + const root = await tempRoot(); + const sourcePath = path.join(root, "source"); + await fs.writeFile(sourcePath, "range-payload"); + const source = await fs.open(sourcePath, "r"); + const directory = await fs.open(root, fsSync.constants.O_RDONLY | fsSync.constants.O_DIRECTORY); + try { + const copied = await native!.copyFileRangeExclusive(source.fd, directory.fd, "target"); + expect(copied.bytes).toBe(13); + fsSync.closeSync(copied.fd); + await expect(fs.readFile(path.join(root, "target"), "utf8")).resolves.toBe("range-payload"); + } finally { + await source.close(); + await directory.close(); + } + }); + + it.runIf(process.platform === "darwin")( + "falls back for ACL-bearing sources without inheriting the ACL", + async () => { + const root = await tempRoot(); + const sourcePath = path.join(root, "source-acl"); + const targetPath = path.join(root, "target-acl"); + await fs.writeFile(sourcePath, "private", { mode: 0o600 }); + execFileSync("chmod", ["+a", "everyone allow read", sourcePath]); + __setNativeLoaderForTest(() => ({ + ...native!, + linkBeneath() { + throw Object.assign(new Error("force copy"), { code: "EXDEV" }); + }, + })); + configureFsSafeNative({ mode: "require" }); + await publishFileExclusive({ sourcePath, targetPath, strategy: "link-or-copy" }); + expect(execFileSync("ls", ["-lde", targetPath], { encoding: "utf8" })).not.toContain( + "everyone:allow read", + ); + }, + ); +}); + +const publishBackends = native + ? (["native", "javascript"] as const) + : (["javascript"] as const); + +describe.each(publishBackends)("%s publication fallback", (backend) => { + it("publishes identical bytes with an exclusive 0600 target", async () => { + const root = await tempRoot(); + const sourcePath = path.join(root, "source"); + const targetPath = path.join(root, "target"); + const payload = Buffer.alloc(256 * 1024, 0x5a); + await fs.writeFile(sourcePath, payload, { mode: 0o644 }); + + if (backend === "native") { + __setNativeLoaderForTest(() => ({ + ...native!, + linkBeneath() { + throw Object.assign(new Error("force copy"), { code: "EXDEV" }); + }, + })); + configureFsSafeNative({ mode: "require" }); + } else { + configureFsSafeNative({ mode: "off" }); + vi.spyOn(fs, "link").mockRejectedValue(Object.assign(new Error("force copy"), { code: "EXDEV" })); + } + + const result = await publishFileExclusive({ + sourcePath, + targetPath, + strategy: "link-or-copy", + }); + expect(result.method).toBe("exclusive-copy"); + await expect(fs.readFile(targetPath)).resolves.toEqual(payload); + if (process.platform !== "win32") { + expect((await fs.stat(targetPath)).mode & 0o777).toBe(0o600); + } + }); +}); diff --git a/test/owner-dacl.test.ts b/test/owner-dacl.test.ts new file mode 100644 index 0000000..747764c --- /dev/null +++ b/test/owner-dacl.test.ts @@ -0,0 +1,91 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { readOwnerAndDacl } from "../src/owner-dacl.js"; +import { __resetFsSafeNativeConfigForTest } from "../src/native-config.js"; +import { + __resetNativeLoaderForTest, + __setNativeLoaderForTest, + type NativeBinding, +} from "../src/native.js"; + +function withProcessPlatform(platform: NodeJS.Platform, body: () => T): T { + const descriptor = Object.getOwnPropertyDescriptor(process, "platform"); + if (!descriptor) throw new Error("process.platform descriptor missing"); + Object.defineProperty(process, "platform", { ...descriptor, value: platform }); + try { + return body(); + } finally { + Object.defineProperty(process, "platform", descriptor); + } +} + +afterEach(() => { + __resetFsSafeNativeConfigForTest(); + __resetNativeLoaderForTest(); +}); + +describe("readOwnerAndDacl", () => { + it.runIf(process.platform !== "win32")("returns a typed unsupported result off Windows", () => { + expect(readOwnerAndDacl("/tmp/example")).toEqual({ + status: "unsupported-platform", + platform: process.platform, + }); + }); + + it.runIf(process.platform !== "win32")("returns ACE facts without applying trust policy", () => { + __setNativeLoaderForTest( + () => + ({ + readOwnerAndDacl: () => ({ + ownerSid: "s-1-5-21-owner", + currentUserSid: "s-1-5-21-current-user", + ownerClass: "foreign", + worldWritable: true, + groupWritable: true, + worldReadable: true, + groupReadable: true, + fallbackRequired: false, + daclPresent: true, + isLocal: true, + aceListComplete: true, + unsupportedAceTypes: [], + aces: [ + { + sid: "s-1-5-21-trusted", + mask: 0x120089, + aceType: "allow", + flags: { + raw: 8, + objectInherit: false, + containerInherit: false, + noPropagateInherit: false, + inheritOnly: true, + inherited: false, + successfulAccess: false, + failedAccess: false, + }, + }, + ], + }), + }) as unknown as NativeBinding, + ); + + const result = withProcessPlatform("win32", () => readOwnerAndDacl("C:\\staging")); + expect(result).toEqual({ + status: "supported", + ownerSid: "s-1-5-21-owner", + currentUserSid: "s-1-5-21-current-user", + daclPresent: true, + isLocal: true, + complete: true, + unsupportedAceTypes: [], + aces: [ + expect.objectContaining({ + sid: "s-1-5-21-trusted", + mask: 0x120089, + aceType: "allow", + flags: expect.objectContaining({ inheritOnly: true }), + }), + ], + }); + }); +}); diff --git a/test/pinned-python.test.ts b/test/pinned-python.test.ts deleted file mode 100644 index c47db6c..0000000 --- a/test/pinned-python.test.ts +++ /dev/null @@ -1,321 +0,0 @@ -import { EventEmitter } from "node:events"; -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import { configureFsSafePython, root } from "../src/index.js"; -import { - canFallbackFromPythonError, - getFsSafePythonConfig, -} from "../src/pinned-python-config.js"; -import { - __resetPinnedPythonWorkerForTest, - isPinnedHelperUnavailable, - runPinnedPythonOperation, -} from "../src/pinned-python.js"; -import * as configSubpath from "../src/config.js"; - -const spawnMock = vi.hoisted(() => vi.fn()); - -vi.mock("node:child_process", () => ({ - spawn: spawnMock, -})); - -type FakeChild = EventEmitter & { - kill: ReturnType; - ref: ReturnType; - stderr: EventEmitter & { - ref: ReturnType; - setEncoding: ReturnType; - unref: ReturnType; - }; - stdin: EventEmitter & { - ref: ReturnType; - unref: ReturnType; - write: ReturnType; - }; - stdout: EventEmitter & { - ref: ReturnType; - setEncoding: ReturnType; - unref: ReturnType; - }; - unref: ReturnType; -}; - -const tempDirs = new Set(); -const originalEnv = { - FS_SAFE_PYTHON: process.env.FS_SAFE_PYTHON, - FS_SAFE_PYTHON_MODE: process.env.FS_SAFE_PYTHON_MODE, - OPENCLAW_FS_SAFE_PYTHON: process.env.OPENCLAW_FS_SAFE_PYTHON, - OPENCLAW_FS_SAFE_PYTHON_MODE: process.env.OPENCLAW_FS_SAFE_PYTHON_MODE, - OPENCLAW_PINNED_PYTHON: process.env.OPENCLAW_PINNED_PYTHON, - OPENCLAW_PINNED_WRITE_PYTHON: process.env.OPENCLAW_PINNED_WRITE_PYTHON, -}; - -function restoreEnv(): void { - for (const [key, value] of Object.entries(originalEnv)) { - if (value === undefined) { - delete process.env[key]; - } else { - process.env[key] = value; - } - } -} - -async function tempRoot(prefix: string): Promise { - const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); - tempDirs.add(dir); - return dir; -} - -function makeChild( - write?: (line: string, callback?: (error?: Error | null) => void) => void, -): FakeChild { - const child = new EventEmitter() as FakeChild; - child.ref = vi.fn(); - child.unref = vi.fn(); - child.kill = vi.fn(); - child.stdout = Object.assign(new EventEmitter(), { - ref: vi.fn(), - setEncoding: vi.fn(), - unref: vi.fn(), - }); - child.stderr = Object.assign(new EventEmitter(), { - ref: vi.fn(), - setEncoding: vi.fn(), - unref: vi.fn(), - }); - child.stdin = Object.assign(new EventEmitter(), { - ref: vi.fn(), - unref: vi.fn(), - write: vi.fn((line: string, callback?: (error?: Error | null) => void) => { - write?.(line, callback); - callback?.(); - return true; - }), - }); - return child; -} - -function makeRespondingChild(): FakeChild { - const child = makeChild((line) => { - const request = JSON.parse(line) as { id: number }; - queueMicrotask(() => { - child.stdout.emit( - "data", - `${JSON.stringify({ id: request.id, ok: true, result: { ok: true } })}\n`, - ); - }); - }); - return child; -} - -function makeFailingChild(): FakeChild { - const child = makeChild(); - queueMicrotask(() => { - const error = Object.assign(new Error("spawn ENOENT"), { - code: "ENOENT", - syscall: "spawn python3", - }); - child.emit("error", error); - }); - return child; -} - -afterEach(async () => { - configureFsSafePython({ mode: "auto", pythonPath: undefined }); - __resetPinnedPythonWorkerForTest(); - restoreEnv(); - spawnMock.mockReset(); - for (const dir of tempDirs) { - await fs.rm(dir, { recursive: true, force: true }); - } - tempDirs.clear(); -}); - -describe("Python helper configuration", () => { - it("reads environment mode and python path aliases", () => { - delete process.env.FS_SAFE_PYTHON_MODE; - delete process.env.OPENCLAW_FS_SAFE_PYTHON_MODE; - delete process.env.FS_SAFE_PYTHON; - delete process.env.OPENCLAW_FS_SAFE_PYTHON; - delete process.env.OPENCLAW_PINNED_PYTHON; - delete process.env.OPENCLAW_PINNED_WRITE_PYTHON; - - expect(getFsSafePythonConfig()).toEqual({ mode: "auto", pythonPath: undefined }); - - process.env.FS_SAFE_PYTHON_MODE = "off"; - process.env.FS_SAFE_PYTHON = "/tmp/python-a"; - expect(getFsSafePythonConfig()).toEqual({ mode: "off", pythonPath: "/tmp/python-a" }); - - delete process.env.FS_SAFE_PYTHON_MODE; - delete process.env.FS_SAFE_PYTHON; - process.env.OPENCLAW_FS_SAFE_PYTHON_MODE = "required"; - process.env.OPENCLAW_PINNED_WRITE_PYTHON = "/tmp/python-b"; - expect(getFsSafePythonConfig()).toEqual({ - mode: "require", - pythonPath: "/tmp/python-b", - }); - - configureFsSafePython({ mode: "auto", pythonPath: "/tmp/python-c" }); - expect(getFsSafePythonConfig()).toEqual({ mode: "auto", pythonPath: "/tmp/python-c" }); - expect(configSubpath.getFsSafePythonConfig()).toEqual({ - mode: "auto", - pythonPath: "/tmp/python-c", - }); - }); - - it("only allows helper fallback errors outside require mode", () => { - const unavailable = Object.assign(new Error("missing"), { code: "helper-unavailable" }); - const unsupportedPlatform = Object.assign(new Error("unsupported"), { code: "unsupported-platform" }); - - configureFsSafePython({ mode: "auto" }); - expect(canFallbackFromPythonError(unavailable)).toBe(true); - expect(canFallbackFromPythonError(unsupportedPlatform)).toBe(true); - - configureFsSafePython({ mode: "off" }); - expect(canFallbackFromPythonError(unavailable)).toBe(true); - expect(canFallbackFromPythonError(unsupportedPlatform)).toBe(true); - - configureFsSafePython({ mode: "require" }); - expect(canFallbackFromPythonError(unavailable)).toBe(false); - expect(canFallbackFromPythonError(unsupportedPlatform)).toBe(false); - }); -}); - -describe("persistent Python helper worker", () => { - it("reuses one worker and unreferences it while idle", async () => { - if (process.platform === "win32") { - configureFsSafePython({ mode: "auto", pythonPath: "/tmp/fake-python" }); - await expect( - runPinnedPythonOperation<{ ok: boolean }>({ - operation: "stat", - rootPath: "/tmp/root", - payload: { relativePath: "a.txt" }, - }), - ).rejects.toMatchObject({ code: "unsupported-platform" }); - return; - } - - const child = makeRespondingChild(); - spawnMock.mockReturnValue(child); - configureFsSafePython({ mode: "auto", pythonPath: "/tmp/fake-python" }); - - await expect( - runPinnedPythonOperation<{ ok: boolean }>({ - operation: "stat", - rootPath: "/tmp/root", - payload: { relativePath: "a.txt" }, - }), - ).resolves.toEqual({ ok: true }); - await expect( - runPinnedPythonOperation<{ ok: boolean }>({ - operation: "stat", - rootPath: "/tmp/root", - payload: { relativePath: "b.txt" }, - }), - ).resolves.toEqual({ ok: true }); - - expect(spawnMock).toHaveBeenCalledTimes(1); - expect(child.ref).toHaveBeenCalled(); - expect(child.unref).toHaveBeenCalled(); - expect(child.stdin.write).toHaveBeenCalledTimes(2); - }); - - it("falls back in auto mode but fails closed in require mode", async () => { - const rootDir = await tempRoot("fs-safe-python-policy-"); - await fs.writeFile(path.join(rootDir, "file.txt"), "ok"); - - spawnMock.mockImplementation(makeFailingChild); - configureFsSafePython({ mode: "auto", pythonPath: "/tmp/missing-python" }); - const autoRoot = await root(rootDir); - await expect(autoRoot.stat("file.txt")).resolves.toMatchObject({ - isFile: true, - }); - await expect(autoRoot.list("")).resolves.toEqual(["file.txt"]); - await fs.writeFile(path.join(rootDir, "move.txt"), "move"); - await autoRoot.move("move.txt", "moved.txt"); - await expect(fs.readFile(path.join(rootDir, "moved.txt"), "utf8")).resolves.toBe("move"); - - __resetPinnedPythonWorkerForTest(); - spawnMock.mockClear(); - spawnMock.mockImplementation(makeFailingChild); - configureFsSafePython({ mode: "require", pythonPath: "/tmp/missing-python" }); - await expect((await root(rootDir)).stat("file.txt")).rejects.toMatchObject({ - code: process.platform === "win32" ? "unsupported-platform" : "helper-unavailable", - }); - }); - - it.runIf(process.platform !== "win32")("ignores stale events from a replaced worker", async () => { - const oldChild = makeChild(); - const newChild = makeChild((line) => { - const request = JSON.parse(line) as { id: number }; - queueMicrotask(() => { - newChild.stdout.emit( - "data", - `${JSON.stringify({ id: request.id, ok: true, result: { ok: true } })}\n`, - ); - }); - }); - spawnMock.mockReturnValueOnce(oldChild).mockReturnValueOnce(newChild); - configureFsSafePython({ mode: "auto", pythonPath: "/tmp/fake-python" }); - - const first = runPinnedPythonOperation({ - operation: "stat", - rootPath: "/tmp/root", - payload: { relativePath: "a.txt" }, - }); - oldChild.emit( - "error", - Object.assign(new Error("old failed"), { code: "ENOENT", syscall: "spawn python3" }), - ); - await expect(first).rejects.toMatchObject({ code: "helper-unavailable" }); - - const second = runPinnedPythonOperation({ - operation: "stat", - rootPath: "/tmp/root", - payload: { relativePath: "b.txt" }, - }); - oldChild.emit("close", 1, null); - - await expect(second).resolves.toEqual({ ok: true }); - }); - - it("classifies helper unavailable errors", () => { - expect(isPinnedHelperUnavailable(Object.assign(new Error("missing"), { - code: "helper-unavailable", - }))).toBe(true); - expect(isPinnedHelperUnavailable(Object.assign(new Error("other"), { - code: "helper-failed", - }))).toBe(false); - }); - - it("maps root identity mismatches", async () => { - const child = makeChild((line) => { - const request = JSON.parse(line) as { id: number }; - queueMicrotask(() => { - child.stdout.emit( - "data", - `${JSON.stringify({ - code: "RuntimeError", - id: request.id, - message: "fs-safe-root-mismatch", - ok: false, - })}\n`, - ); - }); - }); - spawnMock.mockReturnValue(child); - configureFsSafePython({ mode: "auto", pythonPath: "/tmp/fake-python" }); - - await expect( - runPinnedPythonOperation({ - operation: "stat", - rootPath: "/tmp/root", - payload: { relativePath: "" }, - }), - ).rejects.toMatchObject({ - code: process.platform === "win32" ? "unsupported-platform" : "path-mismatch", - }); - }); -}); diff --git a/test/pinned-write-fallback-coverage.test.ts b/test/pinned-write-fallback-coverage.test.ts index d24bcee..4a30dc6 100644 --- a/test/pinned-write-fallback-coverage.test.ts +++ b/test/pinned-write-fallback-coverage.test.ts @@ -44,11 +44,11 @@ afterEach(async () => { describe("pinned write fallback coverage", () => { it.runIf(process.platform !== "win32")( - "writes buffers, creates only when missing, streams, and enforces limits when Python is off", + "writes buffers, creates only when missing, streams, and enforces limits when native mode is off", async () => { - const { configureFsSafePython } = await import("../src/pinned-python-config.js"); + const { configureFsSafeNative } = await import("../src/native-config.js"); const { runPinnedWriteHelper } = await import("../src/pinned-write.js"); - configureFsSafePython({ mode: "off" }); + configureFsSafeNative({ mode: "off" }); const root = await tempRoot("fs-safe-pinned-write-fallback-"); const created = await runPinnedWriteHelper({ @@ -128,10 +128,10 @@ describe("pinned write fallback coverage", () => { }, ); - it.runIf(process.platform !== "win32")("falls back on POSIX when the helper is unavailable in auto mode", async () => { - const { configureFsSafePython } = await import("../src/pinned-python-config.js"); + it.runIf(process.platform !== "win32")("uses the guarded fallback when native mode is off", async () => { + const { configureFsSafeNative } = await import("../src/native-config.js"); const { runPinnedWriteHelper } = await import("../src/pinned-write.js"); - configureFsSafePython({ mode: "auto", pythonPath: "/missing/fs-safe-python" }); + configureFsSafeNative({ mode: "off" }); const root = await tempRoot("fs-safe-pinned-write-fallback-"); await expect( diff --git a/test/pinned-write-fsync.test.ts b/test/pinned-write-fsync.test.ts index eed95ae..1bc5f67 100644 --- a/test/pinned-write-fsync.test.ts +++ b/test/pinned-write-fsync.test.ts @@ -2,8 +2,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { configureFsSafePython, root as openRoot } from "../src/index.js"; -import { __resetPinnedPythonWorkerForTest } from "../src/pinned-python.js"; +import { configureFsSafeNative } from "../src/index.js"; import { runPinnedWriteHelper } from "../src/pinned-write.js"; const tempDirs = new Set(); @@ -16,8 +15,7 @@ async function tempRoot(prefix: string): Promise { afterEach(async () => { vi.restoreAllMocks(); - configureFsSafePython({ mode: "auto", pythonPath: undefined }); - __resetPinnedPythonWorkerForTest(); + configureFsSafeNative({ mode: "auto" }); await Promise.all([...tempDirs].map((dir) => fs.rm(dir, { recursive: true, force: true }))); tempDirs.clear(); }); @@ -26,7 +24,7 @@ describe("pinned write fsync compatibility", () => { it.runIf(process.platform !== "win32")( "treats EPERM from fallback file sync as best effort", async () => { - configureFsSafePython({ mode: "off" }); + configureFsSafeNative({ mode: "off" }); const root = await tempRoot("fs-safe-pinned-write-fsync-eperm-"); const realOpen = fs.open.bind(fs); vi.spyOn(fs, "open").mockImplementation(async (...args) => { @@ -51,42 +49,4 @@ describe("pinned write fsync compatibility", () => { await expect(fs.readFile(path.join(root, "created.txt"), "utf8")).resolves.toBe("created"); }, ); - - it.runIf(process.platform !== "win32")( - "treats EPERM from Python helper file sync as best effort", - async () => { - const rootDir = await tempRoot("fs-safe-python-fsync-eperm-"); - const wrapperPath = path.join(rootDir, "python-wrapper.mjs"); - await fs.writeFile( - wrapperPath, - `#!/usr/bin/env node -import { spawn } from "node:child_process"; - -const args = process.argv.slice(2); -const sourceIndex = args.indexOf("-c") + 1; -if (sourceIndex <= 0 || sourceIndex >= args.length) process.exit(64); -const source = args[sourceIndex]; -args[sourceIndex] = source.replace( - " try: os.fsync(fd)", - " try: raise OSError(errno.EPERM, 'test fsync permission failure')", -); -if (args[sourceIndex] === source) process.exit(65); -const child = spawn("python3", args, { stdio: "inherit" }); -child.on("exit", (code, signal) => { - if (signal) process.kill(process.pid, signal); - process.exit(code ?? 1); -}); -`, - { mode: 0o700 }, - ); - await fs.chmod(wrapperPath, 0o700); - configureFsSafePython({ mode: "require", pythonPath: wrapperPath }); - const scoped = await openRoot(rootDir); - - await expect(scoped.write("created.txt", "payload")).resolves.toBeUndefined(); - await expect(fs.readFile(path.join(rootDir, "created.txt"), "utf8")).resolves.toBe( - "payload", - ); - }, - ); }); diff --git a/test/private-directory.test.ts b/test/private-directory.test.ts new file mode 100644 index 0000000..f3fa9bc --- /dev/null +++ b/test/private-directory.test.ts @@ -0,0 +1,98 @@ +import { createRequire } from "node:module"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { configureFsSafeNative, __resetFsSafeNativeConfigForTest } from "../src/native-config.js"; +import { + __resetNativeLoaderForTest, + __setNativeLoaderForTest, + type NativeBinding, +} from "../src/native.js"; +import { inspectPathPermissions } from "../src/permissions.js"; +import { createPrivateDirectory } from "../src/private-directory.js"; + +const require = createRequire(import.meta.url); +let native: NativeBinding | undefined; +try { + native = require("../native") as NativeBinding; +} catch { + // Ordinary JS jobs do not build a host binding. +} +const tempDirs: string[] = []; + +async function tempRoot(): Promise { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "fs-safe-private-dir-")); + tempDirs.push(root); + return root; +} + +afterEach(async () => { + vi.restoreAllMocks(); + __resetFsSafeNativeConfigForTest(); + __resetNativeLoaderForTest(); + await Promise.all(tempDirs.splice(0).map((root) => fs.rm(root, { recursive: true, force: true }))); +}); + +describe("createPrivateDirectory", () => { + it.runIf(process.platform !== "win32")("fails closed without mutating POSIX paths", async () => { + const root = await tempRoot(); + const target = path.join(root, "private"); + await expect(createPrivateDirectory(target)).rejects.toMatchObject({ + code: "helper-unavailable", + }); + await expect(fs.stat(target)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("fails closed when Windows native mode is off", async () => { + const root = await tempRoot(); + const target = path.join(root, "fallback"); + configureFsSafeNative({ mode: "off" }); + await expect(createPrivateDirectory(target, { platform: "win32" })).rejects.toMatchObject({ + code: "helper-unavailable", + }); + await expect(fs.stat(target)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it.runIf(process.platform === "win32" && Boolean(native))( + "creates and inspects the direct native DACL", + async () => { + const root = await tempRoot(); + const target = path.join(root, "native"); + __setNativeLoaderForTest(() => native!); + configureFsSafeNative({ mode: "require" }); + await createPrivateDirectory(target); + const facts = native!.readOwnerAndDacl(target); + expect(facts.ownerClass).toBe("current-user"); + expect(facts.currentUserSid).toMatch(/^s-/); + expect(facts.ownerSid).toBe(facts.currentUserSid); + expect(facts).toMatchObject({ + worldWritable: false, + groupWritable: false, + fallbackRequired: false, + daclPresent: true, + isLocal: true, + aceListComplete: true, + }); + expect(facts.aces).toHaveLength(3); + expect(facts.aces).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + aceType: "allow", + flags: expect.objectContaining({ + objectInherit: true, + containerInherit: true, + inheritOnly: false, + }), + }), + ]), + ); + await expect(inspectPathPermissions(target)).resolves.toMatchObject({ + source: "windows-acl", + ownerTrusted: true, + worldWritable: false, + groupWritable: false, + }); + }, + ); +}); diff --git a/test/rename-identity-policy.test.ts b/test/rename-identity-policy.test.ts index 26d6626..50f4eb9 100644 --- a/test/rename-identity-policy.test.ts +++ b/test/rename-identity-policy.test.ts @@ -3,8 +3,7 @@ import { createHash } from "node:crypto"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { configureFsSafePython, root as openRoot } from "../src/index.js"; -import { __resetPinnedPythonWorkerForTest } from "../src/pinned-python.js"; +import { configureFsSafeNative, root as openRoot } from "../src/index.js"; import { __setFsSafeTestHooksForTest } from "../src/test-hooks.js"; const tempDirs: string[] = []; @@ -33,8 +32,7 @@ function compatibilityLockPath(rootDir: string, relativePath = "file.txt"): stri afterEach(async () => { __setFsSafeTestHooksForTest(); - __resetPinnedPythonWorkerForTest(); - configureFsSafePython({ mode: "auto", pythonPath: undefined }); + configureFsSafeNative({ mode: "auto" }); for (const dir of tempDirs.splice(0)) { await fsp.rm(dir, { recursive: true, force: true }); } @@ -44,7 +42,7 @@ describe("rename identity policy", () => { it.runIf(process.platform !== "win32")( "keeps strict post-rename identity verification as the default", async () => { - configureFsSafePython({ mode: "off" }); + configureFsSafeNative({ mode: "off" }); const rootDir = await makeTempRoot("fs-safe-rename-id-strict-"); replaceTargetAfterFallbackRename(); @@ -58,7 +56,7 @@ describe("rename identity policy", () => { it.runIf(process.platform !== "win32")( "accepts matching content under the explicit compatibility lock", async () => { - configureFsSafePython({ mode: "off" }); + configureFsSafeNative({ mode: "off" }); const rootDir = await makeTempRoot("fs-safe-rename-id-lock-"); const targetPath = path.join(rootDir, "file.txt"); replaceTargetAfterFallbackRename(); @@ -75,7 +73,7 @@ describe("rename identity policy", () => { it.runIf(process.platform !== "win32")( "rejects replacement content despite the compatibility lock", async () => { - configureFsSafePython({ mode: "off" }); + configureFsSafeNative({ mode: "off" }); const rootDir = await makeTempRoot("fs-safe-rename-id-attack-"); replaceTargetAfterFallbackRename("attacker-content"); @@ -89,7 +87,7 @@ describe("rename identity policy", () => { it.runIf(process.platform !== "win32")( "supports a per-call compatibility override", async () => { - configureFsSafePython({ mode: "off" }); + configureFsSafeNative({ mode: "off" }); const rootDir = await makeTempRoot("fs-safe-rename-id-per-call-"); replaceTargetAfterFallbackRename(); @@ -103,10 +101,10 @@ describe("rename identity policy", () => { ); it.runIf(process.platform !== "win32")( - "deliberately routes compatibility writes around required Python mode", + "deliberately keeps compatibility writes on the guarded JavaScript path", async () => { - configureFsSafePython({ mode: "require" }); - const rootDir = await makeTempRoot("fs-safe-rename-id-python-"); + configureFsSafeNative({ mode: "off" }); + const rootDir = await makeTempRoot("fs-safe-rename-id-native-"); replaceTargetAfterFallbackRename(); const fs = await openRoot(rootDir, { renameIdentity: "verify-content-with-lock" }); @@ -118,7 +116,7 @@ describe("rename identity policy", () => { it.runIf(process.platform !== "win32")( "preserves already-exists errors for create", async () => { - configureFsSafePython({ mode: "off" }); + configureFsSafeNative({ mode: "off" }); const rootDir = await makeTempRoot("fs-safe-rename-id-create-"); await fsp.writeFile(path.join(rootDir, "file.txt"), "existing"); @@ -132,7 +130,7 @@ describe("rename identity policy", () => { it.runIf(process.platform !== "win32")( "does not create a missing parent when mkdir is disabled", async () => { - configureFsSafePython({ mode: "off" }); + configureFsSafeNative({ mode: "off" }); const rootDir = await makeTempRoot("fs-safe-rename-id-no-mkdir-"); const parentPath = path.join(rootDir, "missing"); @@ -163,7 +161,7 @@ describe("rename identity policy", () => { it.runIf(process.platform !== "win32")( "works on a normal POSIX filesystem and releases the lock", async () => { - configureFsSafePython({ mode: "off" }); + configureFsSafeNative({ mode: "off" }); const rootDir = await makeTempRoot("fs-safe-rename-id-posix-"); const fs = await openRoot(rootDir, { renameIdentity: "verify-content-with-lock" }); diff --git a/test/root-walk-options.test.ts b/test/root-walk-options.test.ts new file mode 100644 index 0000000..4c6dd08 --- /dev/null +++ b/test/root-walk-options.test.ts @@ -0,0 +1,125 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { root } from "../src/root.js"; +import { walkRoot, type RootWalkEntry } from "../src/root-walk.js"; +import type { DirEntry } from "../src/types.js"; + +const tempDirs: string[] = []; + +async function tempRoot(): Promise { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "fs-safe-root-walk-options-")); + tempDirs.push(directory); + return directory; +} + +afterEach(async () => { + await Promise.all( + tempDirs.splice(0).map((directory) => fs.rm(directory, { recursive: true, force: true })), + ); +}); + +it("prunes skip-subtree directories while plain skip still descends", async () => { + const directory = await tempRoot(); + await fs.mkdir(path.join(directory, "keep")); + await fs.mkdir(path.join(directory, "skip")); + await fs.writeFile(path.join(directory, "keep", "value.txt"), "keep"); + await fs.writeFile(path.join(directory, "skip", "hidden.txt"), "skip"); + const capability = await root(directory); + + const pruned: string[] = []; + for await (const entry of capability.walk("", { + symlinkPolicy: "skip", + entryFilter: (entry) => + entry.kind === "directory" && entry.relativePath === "skip" + ? "skip-subtree" + : "include", + })) { + pruned.push(entry.relativePath); + } + expect(pruned).toContain("keep/value.txt"); + expect(pruned).not.toContain("skip"); + expect(pruned).not.toContain("skip/hidden.txt"); + + const filtered: string[] = []; + for await (const entry of capability.walk("", { + symlinkPolicy: "skip", + entryFilter: (entry) => (entry.relativePath === "skip" ? "skip" : "include"), + })) { + filtered.push(entry.relativePath); + } + expect(filtered).not.toContain("skip"); + expect(filtered).toContain("skip/hidden.txt"); +}); + +it("counts skipped entries against the traversal budget", async () => { + const directory = await tempRoot(); + await fs.writeFile(path.join(directory, "one.txt"), "one"); + await fs.writeFile(path.join(directory, "two.txt"), "two"); + const capability = await root(directory); + const filtered: string[] = []; + const entries: RootWalkEntry[] = []; + + for await (const entry of capability.walk("", { + maxEntries: 1, + symlinkPolicy: "skip", + entryFilter: (candidate) => { + filtered.push(candidate.relativePath); + return "skip"; + }, + })) { + entries.push(entry); + } + + expect(filtered).toHaveLength(1); + expect(entries).toEqual([ + { relativePath: expect.any(String), kind: "truncated", size: 0 }, + ]); +}); + +it("reports failed directory subtrees and continues when requested", async () => { + const directory = await tempRoot(); + await fs.mkdir(path.join(directory, "broken")); + await fs.mkdir(path.join(directory, "healthy")); + await fs.writeFile(path.join(directory, "healthy", "value.txt"), "healthy"); + const capability = await root(directory); + const list = capability.list.bind(capability) as ( + relativePath: string, + options: { withFileTypes: true }, + ) => Promise; + const walkingRoot = { + rootReal: capability.rootReal, + async list(relativePath: string, options: { withFileTypes: true }): Promise { + if (relativePath === "broken") { + throw Object.assign(new Error("unreadable subtree"), { code: "EACCES" }); + } + return await list(relativePath, options); + }, + }; + + const entries: RootWalkEntry[] = []; + for await (const entry of walkRoot(walkingRoot, "", { + symlinkPolicy: "skip", + onDirectoryError: "skip-and-report", + })) { + entries.push(entry); + } + expect(entries).toContainEqual({ + relativePath: "broken", + kind: "directory-error", + size: 0, + error: expect.objectContaining({ code: "EACCES" }), + }); + expect(entries).toContainEqual({ + relativePath: "healthy/value.txt", + kind: "file", + size: 7, + }); + + await expect(async () => { + for await (const _entry of walkRoot(walkingRoot, "", { symlinkPolicy: "skip" })) { + // Consume the iterator to prove the default remains fail-fast. + } + }).rejects.toMatchObject({ code: "EACCES" }); +}); diff --git a/test/sidecar-lock-ownership-token.test.ts b/test/sidecar-lock-ownership-token.test.ts index edfeaa0..c7118e1 100644 --- a/test/sidecar-lock-ownership-token.test.ts +++ b/test/sidecar-lock-ownership-token.test.ts @@ -4,6 +4,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { createSidecarLockManager } from "../src/sidecar-lock.js"; +import { configureFsSafeNative } from "../src/native-config.js"; import { readSidecarLockOwnershipToken, readSidecarLockSnapshot, @@ -20,6 +21,7 @@ async function tempRoot(prefix: string): Promise { } afterEach(async () => { + configureFsSafeNative({ mode: "auto" }); vi.restoreAllMocks(); await Promise.all(tempDirs.splice(0).map((dir) => fsp.rm(dir, { recursive: true, force: true }))); }); @@ -282,6 +284,7 @@ describe("sidecar lock ownership tokens", () => { }); it("cleans a partial sidecar left by a failed write", async () => { + configureFsSafeNative({ mode: "off" }); const base = await tempRoot("fs-safe-sidecar-partial-write-"); const targetPath = path.join(base, "state.json"); const lockPath = `${targetPath}.lock`; diff --git a/test/stage1-primitives.test.ts b/test/stage1-primitives.test.ts new file mode 100644 index 0000000..b5743fb --- /dev/null +++ b/test/stage1-primitives.test.ts @@ -0,0 +1,428 @@ +import fsSync from "node:fs"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + isHardlinkFallbackError, + publishFileExclusive, +} from "../src/durability.js"; +import { + acquireFileLock, + acquireFileLockSync, + withFileLockSync, +} from "../src/file-lock.js"; +import { root } from "../src/root.js"; +import { + createSecretFileAtomic, + readSecretFile, + tryReadSecretFile, +} from "../src/secret.js"; +import { tempWorkspace } from "../src/temp.js"; +import { configureFsSafeNative } from "../src/native-config.js"; +import { FsSafeError } from "../src/errors.js"; +import { __setFsSafeTestHooksForTest } from "../src/test-hooks.js"; + +const tempDirs: string[] = []; + +async function tempRoot(prefix: string): Promise { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); + tempDirs.push(directory); + return directory; +} + +afterEach(async () => { + configureFsSafeNative({ mode: "auto" }); + __setFsSafeTestHooksForTest(); + vi.restoreAllMocks(); + await Promise.all(tempDirs.splice(0).map((directory) => fs.rm(directory, { recursive: true, force: true }))); +}); + +describe("exclusive file publication", () => { + it("closes the source handle when publication parent pinning fails", async () => { + configureFsSafeNative({ mode: "off" }); + const directory = await tempRoot("fs-safe-publish-parent-failure-"); + const sourcePath = path.join(directory, "source"); + const targetPath = path.join(directory, "missing", "target"); + await fs.writeFile(sourcePath, "content"); + const open = fs.open.bind(fs); + let opened = 0; + let closed = 0; + vi.spyOn(fs, "open").mockImplementationOnce(async (...args) => { + const handle = await open(...args); + opened += 1; + const close = handle.close.bind(handle); + vi.spyOn(handle, "close").mockImplementationOnce(async () => { + closed += 1; + await close(); + }); + return handle; + }); + + await expect( + publishFileExclusive({ sourcePath, targetPath, strategy: "link-required" }), + ).rejects.toMatchObject({ code: "ENOENT" }); + expect({ opened, closed }).toEqual({ opened: 1, closed: 1 }); + }); + + it("publishes by hardlink without clobbering an existing target", async () => { + const directory = await tempRoot("fs-safe-publish-link-"); + const sourcePath = path.join(directory, "source"); + const targetPath = path.join(directory, "target"); + await fs.writeFile(sourcePath, "content"); + const result = await publishFileExclusive({ sourcePath, targetPath, strategy: "link-required" }); + expect(result.method).toBe("hardlink"); + expect(result.directorySync.status).toMatch(/^(?:synced|unsupported)$/u); + await expect(fs.readFile(targetPath, "utf8")).resolves.toBe("content"); + await expect( + publishFileExclusive({ sourcePath, targetPath, strategy: "link-or-copy" }), + ).rejects.toMatchObject({ code: "EEXIST" }); + }); + + it("falls back only for classified hardlink errors and copies from the pinned source", async () => { + configureFsSafeNative({ mode: "off" }); + const directory = await tempRoot("fs-safe-publish-copy-"); + const sourcePath = path.join(directory, "source"); + const targetPath = path.join(directory, "target"); + await fs.writeFile(sourcePath, "copy-content"); + vi.spyOn(fs, "link").mockRejectedValueOnce(Object.assign(new Error("unsupported"), { code: "EXDEV" })); + const result = await publishFileExclusive({ sourcePath, targetPath, strategy: "link-or-copy" }); + expect(result.method).toBe("exclusive-copy"); + await expect(fs.readFile(targetPath, "utf8")).resolves.toBe("copy-content"); + expect(isHardlinkFallbackError(Object.assign(new Error(), { code: "EOPNOTSUPP" }))).toBe(true); + expect(isHardlinkFallbackError(Object.assign(new Error(), { code: "EIO" }))).toBe(false); + }); + + it.each(["removed", "preserved", "unknown"] as const)( + "reports %s cleanup after a post-hardlink failure", + async (cleanup) => { + configureFsSafeNative({ mode: "off" }); + const directory = await tempRoot(`fs-safe-publish-${cleanup}-`); + const sourcePath = path.join(directory, "source"); + const targetPath = path.join(directory, "target"); + const originalLinkPath = path.join(directory, "original-link"); + await fs.writeFile(sourcePath, "source-content"); + + __setFsSafeTestHooksForTest({ + async afterPublishTargetCreated(method, createdPath) { + expect(method).toBe("hardlink"); + expect(createdPath).toBe(targetPath); + if (cleanup === "preserved") { + await fs.rename(targetPath, originalLinkPath); + await fs.writeFile(targetPath, "replacement-content"); + } + throw new Error("post-publication guard failed"); + }, + }); + const rm = + cleanup === "unknown" + ? vi + .spyOn(fs, "rm") + .mockRejectedValueOnce(Object.assign(new Error("cleanup denied"), { code: "EACCES" })) + : undefined; + + try { + await expect( + publishFileExclusive({ sourcePath, targetPath, strategy: "link-required" }), + ).rejects.toSatisfy((error: unknown) => { + expect(error).toBeInstanceOf(FsSafeError); + expect(error).toMatchObject({ + code: "helper-failed", + details: { + phase: "hardlink-verify", + targetCreated: true, + targetIdentity: { dev: expect.any(Number), ino: expect.any(Number) }, + cleanup, + }, + }); + return true; + }); + } finally { + rm?.mockRestore(); + } + + if (cleanup === "removed") { + await expect(fs.access(targetPath)).rejects.toMatchObject({ code: "ENOENT" }); + } else if (cleanup === "preserved") { + await expect(fs.readFile(targetPath, "utf8")).resolves.toBe("replacement-content"); + await expect(fs.readFile(originalLinkPath, "utf8")).resolves.toBe("source-content"); + } else { + await expect(fs.readFile(targetPath, "utf8")).resolves.toBe("source-content"); + } + }, + ); + + it.each([ + ["rollback", "removed"], + ["preserve", "preserved"], + ] as const)( + "%s sync-failure policy reports %s cleanup for an unchanged target", + async (onSyncFailure, cleanup) => { + configureFsSafeNative({ mode: "off" }); + const directory = await tempRoot(`fs-safe-publish-sync-${onSyncFailure}-`); + const sourcePath = path.join(directory, "source"); + const targetPath = path.join(directory, "target"); + await fs.writeFile(sourcePath, "complete-archive"); + __setFsSafeTestHooksForTest({ + beforePublishDirectorySync(method, createdPath) { + expect(method).toBe("hardlink"); + expect(createdPath).toBe(targetPath); + throw Object.assign(new Error("directory sync failed"), { code: "EIO" }); + }, + }); + + await expect( + publishFileExclusive({ + sourcePath, + targetPath, + strategy: "link-required", + onSyncFailure, + }), + ).rejects.toMatchObject({ + code: "helper-failed", + details: { + phase: "directory-sync", + targetCreated: true, + cleanup, + directorySync: { status: "failed", code: "EIO" }, + }, + }); + + if (onSyncFailure === "rollback") { + await expect(fs.access(targetPath)).rejects.toMatchObject({ code: "ENOENT" }); + } else { + await expect(fs.readFile(targetPath, "utf8")).resolves.toBe("complete-archive"); + } + }, + ); + + it("preserves a replacement target when its identity changes during sync failure", async () => { + configureFsSafeNative({ mode: "off" }); + const directory = await tempRoot("fs-safe-publish-sync-drift-"); + const sourcePath = path.join(directory, "source"); + const targetPath = path.join(directory, "target"); + const originalLinkPath = path.join(directory, "original-link"); + await fs.writeFile(sourcePath, "complete-archive"); + __setFsSafeTestHooksForTest({ + async beforePublishDirectorySync() { + await fs.rename(targetPath, originalLinkPath); + await fs.writeFile(targetPath, "replacement"); + throw Object.assign(new Error("target identity changed"), { code: "path-mismatch" }); + }, + }); + + await expect( + publishFileExclusive({ + sourcePath, + targetPath, + strategy: "link-required", + onSyncFailure: "rollback", + }), + ).rejects.toMatchObject({ + details: { + phase: "directory-sync", + targetCreated: true, + cleanup: "preserved", + directorySync: { status: "failed", code: "path-mismatch" }, + }, + }); + await expect(fs.readFile(targetPath, "utf8")).resolves.toBe("replacement"); + await expect(fs.readFile(originalLinkPath, "utf8")).resolves.toBe( + "complete-archive", + ); + }); + + it.runIf(process.platform !== "win32")( + "detects parent replacement inside the shared directory-sync boundary", + async () => { + configureFsSafeNative({ mode: "off" }); + const directory = await tempRoot("fs-safe-publish-parent-drift-"); + const movedDirectory = `${directory}.moved`; + tempDirs.push(movedDirectory); + const sourcePath = path.join(directory, "source"); + const targetPath = path.join(directory, "target"); + await fs.writeFile(sourcePath, "complete-archive"); + __setFsSafeTestHooksForTest({ + async beforePublishDirectorySync() { + await fs.rename(directory, movedDirectory); + await fs.mkdir(directory); + await fs.writeFile(targetPath, "replacement"); + }, + }); + + await expect( + publishFileExclusive({ + sourcePath, + targetPath, + strategy: "link-required", + onSyncFailure: "rollback", + }), + ).rejects.toMatchObject({ + details: { + phase: "directory-sync", + cleanup: "preserved", + directorySync: { status: "failed", code: "path-mismatch" }, + }, + }); + await expect(fs.readFile(targetPath, "utf8")).resolves.toBe("replacement"); + await expect(fs.readFile(path.join(movedDirectory, "target"), "utf8")).resolves.toBe( + "complete-archive", + ); + }, + ); +}); + +describe("secret file additions", () => { + it("creates once and supports strict and try-style async reads", async () => { + const directory = await tempRoot("fs-safe-secret-create-"); + const filePath = path.join(directory, "private", "token"); + await createSecretFileAtomic({ rootDir: directory, filePath, content: " token\n" }); + await expect(readSecretFile(filePath, "token")).resolves.toBe("token"); + await expect(tryReadSecretFile(path.join(directory, "missing"), "token")).resolves.toBeUndefined(); + await expect( + createSecretFileAtomic({ rootDir: directory, filePath, content: "replacement" }), + ).rejects.toMatchObject({ code: "secret-exists" }); + await expect(fs.readFile(filePath, "utf8")).resolves.toBe(" token\n"); + }); +}); + +describe("file lock additions", () => { + it("supports synchronous acquisition without the manager queue", async () => { + const directory = await tempRoot("fs-safe-lock-sync-"); + const targetPath = path.join(directory, "state.json"); + const lock = acquireFileLockSync(targetPath, { + payload: () => ({ createdAt: new Date().toISOString() }), + timeoutMs: 0, + retry: { retries: 0 }, + }); + expect(lock.verifyStillHeld()).toBe(true); + expect(() => + withFileLockSync( + targetPath, + { payload: () => ({}), timeoutMs: 0, retry: { retries: 0 } }, + () => undefined, + ), + ).toThrow(/timeout/u); + lock.release(); + expect(fsSync.existsSync(lock.lockPath)).toBe(false); + }); + + it("bounds async sidecars through Root and reports compromised ownership", async () => { + const directory = await tempRoot("fs-safe-lock-root-"); + const lockDirectory = path.join(directory, "locks"); + await fs.mkdir(lockDirectory); + const lockRoot = await root(lockDirectory); + const lockPath = path.join(lockDirectory, "state.lock"); + let compromised = false; + const lock = await acquireFileLock(path.join(directory, "state.json"), { + lockPath, + lockRoot, + payload: () => ({ createdAt: new Date().toISOString() }), + compromiseCheckIntervalMs: 5, + onCompromised: () => { + compromised = true; + }, + }); + expect(await lock.verifyStillHeld()).toBe(true); + await fs.writeFile(lockPath, "replacement"); + await vi.waitFor(() => expect(compromised).toBe(true)); + expect(await lock.verifyStillHeld()).toBe(false); + await lock.release(); + await expect(fs.readFile(lockPath, "utf8")).resolves.toBe("replacement"); + + await expect( + acquireFileLock(path.join(directory, "other.json"), { + lockPath: path.join(directory, "outside.lock"), + lockRoot, + payload: () => ({}), + }), + ).rejects.toMatchObject({ code: "outside-workspace" }); + }); + + it("lets application parsers drive guarded legacy payload reclaim", async () => { + const directory = await tempRoot("fs-safe-lock-legacy-"); + const targetPath = path.join(directory, "state.json"); + const lockPath = `${targetPath}.lock`; + await fs.writeFile(lockPath, "pid=123\n"); + const seen: unknown[] = []; + const lock = await acquireFileLock(targetPath, { + lockPath, + staleRecovery: "remove-if-unchanged", + staleMs: 1, + payload: () => ({ createdAt: new Date().toISOString() }), + parsePayload: (raw) => ({ pid: Number(raw.match(/\d+/u)?.[0]) }), + shouldReclaim: ({ payload }) => { + seen.push(payload); + return true; + }, + shouldRemoveStaleLock: ({ payload }) => { + seen.push(payload); + return true; + }, + }); + expect(seen).toEqual([{ pid: 123 }, { pid: 123 }]); + await lock.release(); + }); +}); + +describe("root walk and temp receipts", () => { + it("walks within a Root and reports or throws on budgets", async () => { + const directory = await tempRoot("fs-safe-root-walk-"); + await fs.mkdir(path.join(directory, "nested")); + await fs.writeFile(path.join(directory, "nested", "value.txt"), "value"); + const capability = await root(directory); + const entries = []; + for await (const entry of capability.walk("", { + maxDepth: 0, + maxEntries: 10, + symlinkPolicy: "skip", + })) { + entries.push(entry); + } + expect(entries).toEqual([ + { relativePath: "nested", kind: "directory", size: expect.any(Number) }, + { relativePath: "nested", kind: "truncated", size: 0 }, + ]); + await expect(async () => { + for await (const _entry of capability.walk("", { + maxEntries: 0, + symlinkPolicy: "skip", + limitBehavior: "throw", + })) { + // Consume the iterator. + } + }).rejects.toMatchObject({ code: "too-large" }); + }); + + it.runIf(process.platform !== "win32")("follows only symlinks that remain inside the walk root", async () => { + const directory = await tempRoot("fs-safe-root-walk-links-"); + const outside = await tempRoot("fs-safe-root-walk-outside-"); + await fs.mkdir(path.join(directory, "real")); + await fs.writeFile(path.join(directory, "real", "value"), "value"); + await fs.symlink(path.join(directory, "real"), path.join(directory, "inside")); + await fs.symlink(outside, path.join(directory, "outside")); + const capability = await root(directory); + const seen: string[] = []; + await expect(async () => { + for await (const entry of capability.walk("", { + symlinkPolicy: "follow-within-root", + })) { + seen.push(entry.relativePath); + } + }).rejects.toThrow(/root walk/u); + expect(seen).toContain("inside"); + expect(seen).not.toContain("outside/value"); + }); + + it("does not dispose a replacement temp workspace", async () => { + const directory = await tempRoot("fs-safe-temp-receipt-"); + const workspace = await tempWorkspace({ rootDir: directory, prefix: "work-" }); + expect(workspace.identity).toMatchObject({ dev: expect.any(Number), ino: expect.any(Number) }); + const original = `${workspace.dir}.original`; + await fs.rename(workspace.dir, original); + await fs.mkdir(workspace.dir); + await fs.writeFile(path.join(workspace.dir, "replacement"), "keep"); + await expect(workspace.cleanup()).resolves.toBe("identity-mismatch"); + await expect(fs.readFile(path.join(workspace.dir, "replacement"), "utf8")).resolves.toBe("keep"); + }); +}); diff --git a/test/write-boundary-bypass.test.ts b/test/write-boundary-bypass.test.ts index a5d5959..785a33b 100644 --- a/test/write-boundary-bypass.test.ts +++ b/test/write-boundary-bypass.test.ts @@ -3,7 +3,7 @@ import path from "node:path"; import { Readable } from "node:stream"; import { afterEach, describe, expect, it } from "vitest"; import { fileStore, fileStoreSync } from "../src/file-store.js"; -import { configureFsSafePython, root as openRoot } from "../src/index.js"; +import { configureFsSafeNative, root as openRoot } from "../src/index.js"; import { ESCAPING_DIRECTORY_PAYLOADS, ESCAPING_WRITE_PAYLOADS, @@ -24,7 +24,7 @@ async function makeTempLayout(prefix: string) { } afterEach(async () => { - configureFsSafePython({ mode: "auto", pythonPath: undefined }); + configureFsSafeNative({ mode: "auto" }); await Promise.all(tempDirs.splice(0).map((dir) => fsp.rm(dir, { force: true, recursive: true }))); }); @@ -241,7 +241,7 @@ describe("write, move, and delete boundary bypass attempts", () => { }, 15000); it.runIf(process.platform !== "win32")("keeps literal '..'-prefixed read paths available when the helper is disabled", async () => { - configureFsSafePython({ mode: "off" }); + configureFsSafeNative({ mode: "off" }); const layout = await makeTempLayout("fs-safe-write-helper-off-literal"); const safeRoot = await openRoot(layout.root); await fsp.writeFile(path.join(layout.root, "..%2fpwned.txt"), "literal");